diff --git a/backend.go b/backend.go index 6e72b7f..183df52 100644 --- a/backend.go +++ b/backend.go @@ -1,42 +1,51 @@ package imapmaildir import ( - "errors" - "io/ioutil" + "fmt" "log" "os" + "sort" "strings" "sync" + "time" - "github.com/asdine/storm/v3" "github.com/emersion/go-imap" "github.com/emersion/go-imap/backend" + mess "github.com/foxcpp/go-imap-mess" + + "github.com/foxcpp/go-imap-maildir/maildir" ) type Backend struct { Log *log.Logger Debug *log.Logger - PathTemplate string - Authenticator func(*imap.ConnInfo, string, string) (bool, error) - - // BoltDB does not allow to open the same database file multiple times, - // therefore we need to serialize access to one handle and close it only if - // the mailbox is no longer used. - // - // dbsLock protects the concurrent map access. At the moment there is no - // clever locking and dbsLock is held for the whole duration of storm.DB - // initialization. That is, once dbsLock is acquired, all elements in the - // map have vaild db. - // - // Lookup key is username + \0 + mailboxName. - dbs map[string]mailboxHandle - dbsLock sync.Mutex + PathTemplate string + Authenticator func(*imap.ConnInfo, string, string) (bool, error) + DefaultMailboxes []DefaultMailboxSpec + StorageProvider maildir.Provider + + Manager *mess.Manager + + limitLock sync.Mutex + appendLimit *uint32 + + statesLock sync.Mutex + states map[string]*mailboxState +} + +type mailboxState struct { + uidValidity uint32 + uidNext uint32 + messages map[string]*messageMeta + meta *maildir.Metadata } -type mailboxHandle struct { - db *storm.DB - uses int64 +type messageMeta struct { + uid uint32 + flags []string + internalDate time.Time + recent bool } func (b *Backend) Login(connInfo *imap.ConnInfo, username, password string) (backend.User, error) { @@ -55,55 +64,213 @@ func (b *Backend) Login(connInfo *imap.ConnInfo, username, password string) (bac func (b *Backend) GetUser(username string) (backend.User, error) { basePath := strings.ReplaceAll(b.PathTemplate, "{username}", username) - - if _, err := os.Stat(basePath); err != nil { - if os.IsNotExist(err) { - return nil, backend.ErrInvalidCredentials - } + storage := b.StorageProvider.Storage(basePath) + mailbox, err := storage.Dir(InboxName) + if err != nil { b.Log.Printf("%v", err) - return nil, errors.New("I/O error") + return nil, fmt.Errorf("I/O error: %w", err) + } + exists, err := mailbox.Exists() + if err != nil { + b.Log.Printf("%v", err) + return nil, fmt.Errorf("I/O error: %w", err) + } + if !exists { + return nil, backend.ErrInvalidCredentials } b.Debug.Printf("user logged in (%v, %v)", username, basePath) - return &User{ - b: b, - name: username, - basePath: basePath, - }, nil + user := &User{ + b: b, + name: username, + storage: storage, + mailboxes: map[string]*Mailbox{}, + } + if err := mailbox.Init(); err != nil { + b.Log.Printf("failed to init inbox: %v", err) + return nil, fmt.Errorf("I/O error: %w", err) + } + + return user, nil } func (b *Backend) CreateUser(username string) error { basePath := strings.ReplaceAll(b.PathTemplate, "{username}", username) - - err := os.Mkdir(basePath, 0700) + storage := b.StorageProvider.Storage(basePath) + mailbox, err := storage.Dir(InboxName) if err != nil { - if os.IsExist(err) { - return errors.New("imapmaildir: user already exits") - } return err } + if err := mailbox.Init(); err != nil { + return err + } + + user := &User{ + b: b, + name: username, + storage: storage, + mailboxes: map[string]*Mailbox{}, + } + user.ensureMailbox(InboxName) + user.ensureDefaultMailboxes() return nil } -func (b *Backend) Close() error { - b.dbsLock.Lock() - defer b.dbsLock.Unlock() +func (b *Backend) DeleteUser(username string) error { + basePath := strings.ReplaceAll(b.PathTemplate, "{username}", username) + storage := b.StorageProvider.Storage(basePath) + + mailboxes := []string{} + seen := map[string]struct{}{} + addMailbox := func(name string) { + if name == "" { + return + } + if _, ok := seen[name]; ok { + return + } + seen[name] = struct{}{} + mailboxes = append(mailboxes, name) + } + + if inbox, err := storage.Dir(InboxName); err == nil { + exists, err := inbox.Exists() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + if exists { + addMailbox(InboxName) + } + } + + names, err := storage.ListDirs() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + for _, name := range names { + addMailbox(name) + } - for k, db := range b.dbs { - if err := db.db.Close(); err != nil { - b.Log.Printf("close failed for %s DB: %v", k, err) + sort.Slice(mailboxes, func(i, j int) bool { + depthI := strings.Count(mailboxes[i], HierarchySep) + depthJ := strings.Count(mailboxes[j], HierarchySep) + if depthI == depthJ { + return mailboxes[i] > mailboxes[j] } + return depthI > depthJ + }) + + for _, name := range mailboxes { + dir, err := storage.Dir(name) + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + exists, err := dir.Exists() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + if !exists { + continue + } + if err := dir.Remove(false); err != nil { + return fmt.Errorf("I/O error: %w", err) + } + b.deleteMailboxState(username, name) } return nil } -func New(pathTemplate string) (*Backend, error) { +func (b *Backend) CreateMessageLimit() *uint32 { + b.limitLock.Lock() + defer b.limitLock.Unlock() + + if b.appendLimit == nil { + return nil + } + val := *b.appendLimit + return &val +} + +func (b *Backend) SetMessageLimit(val *uint32) error { + b.limitLock.Lock() + defer b.limitLock.Unlock() + + if val == nil { + b.appendLimit = nil + return nil + } + + copyVal := *val + b.appendLimit = ©Val + return nil +} + +func (b *Backend) Close() error { + return nil +} + +func New(pathTemplate string, provider maildir.Provider, defaultMailboxes []DefaultMailboxSpec) (*Backend, error) { + if len(defaultMailboxes) == 0 { + defaultMailboxes = defaultMailboxSpecs + } return &Backend{ - Log: log.New(os.Stderr, "imapmaildir: ", 0), - Debug: log.New(ioutil.Discard, "imapmaildir[debug]: ", 0), - PathTemplate: pathTemplate, - dbs: map[string]mailboxHandle{}, + Log: log.New(os.Stderr, "imapmaildir: ", 0), + Debug: log.New(os.Stderr, "imapmaildir[debug]: ", 0), + PathTemplate: pathTemplate, + DefaultMailboxes: defaultMailboxes, + StorageProvider: provider, + Manager: mess.NewManager(), + states: map[string]*mailboxState{}, }, nil } + +func (b *Backend) mailboxKey(username, mailbox string) string { + return username + "\x00" + mailbox +} + +func (b *Backend) getMailboxState(username, mailbox string) *mailboxState { + key := b.mailboxKey(username, mailbox) + + b.statesLock.Lock() + defer b.statesLock.Unlock() + + state, ok := b.states[key] + if ok { + return state + } + + uidValidity := uint32(time.Now().UnixNano()) + if uidValidity == 0 { + uidValidity = 1 + } + + state = &mailboxState{ + uidValidity: uidValidity, + uidNext: 1, + messages: map[string]*messageMeta{}, + } + b.states[key] = state + return state +} + +func (b *Backend) deleteMailboxState(username, mailbox string) { + key := b.mailboxKey(username, mailbox) + b.statesLock.Lock() + defer b.statesLock.Unlock() + delete(b.states, key) +} + +func (b *Backend) renameMailboxState(username, oldName, newName string) { + oldKey := b.mailboxKey(username, oldName) + newKey := b.mailboxKey(username, newName) + + b.statesLock.Lock() + defer b.statesLock.Unlock() + + if state, ok := b.states[oldKey]; ok { + delete(b.states, oldKey) + b.states[newKey] = state + } +} diff --git a/backendtests_test.go b/backendtests_fs_test.go similarity index 83% rename from backendtests_test.go rename to backendtests_fs_test.go index 842df66..543257f 100644 --- a/backendtests_test.go +++ b/backendtests_fs_test.go @@ -1,22 +1,23 @@ package imapmaildir import ( - "io/ioutil" "log" "os" "strings" "testing" backendtests "github.com/foxcpp/go-imap-backend-tests" + + "github.com/foxcpp/go-imap-maildir/maildir/fs" ) func initTestBackend() backendtests.Backend { - tempDir, err := ioutil.TempDir("", "go-imap-maildir-") + tempDir, err := os.MkdirTemp("", "go-imap-maildir-") if err != nil { panic(err) } - be, err := New(tempDir + "/{username}") + be, err := New(tempDir+"/{username}", fs.Provider{}, nil) if err != nil { panic(err) } diff --git a/backendtests_s3_test.go b/backendtests_s3_test.go new file mode 100644 index 0000000..052041c --- /dev/null +++ b/backendtests_s3_test.go @@ -0,0 +1,51 @@ +package imapmaildir + +import ( + "strings" + "testing" + + backendtests "github.com/foxcpp/go-imap-backend-tests" + + "github.com/foxcpp/go-imap-maildir/maildir/s3" + "github.com/foxcpp/go-imap-maildir/maildir/s3/testing" +) + +func TestBackendS3(t *testing.T) { + client, bucket, rootPrefix, cleanup := s3testing.StartMinio(t) + defer cleanup() + + provider := s3.NewProvider(client, bucket, rootPrefix) + + init := func() backendtests.Backend { + be, err := New("users/{username}", provider, defaultMailboxSpecs) + if err != nil { + panic(err) + } + return be + } + + clean := func(b backendtests.Backend) { + be := b.(*Backend) + prefix := "" + if be.PathTemplate != "" { + prefix = joinKey(rootPrefix, "users") + } + s3testing.RemoveAllObjects(t, client, bucket, prefix) + } + + backendtests.RunTests(t, init, clean) +} + +func joinKey(parts ...string) string { + var clean []string + for _, part := range parts { + if part == "" { + continue + } + clean = append(clean, strings.Trim(part, "/")) + } + if len(clean) == 0 { + return "" + } + return strings.Join(clean, "/") +} diff --git a/delivery.go b/delivery.go new file mode 100644 index 0000000..1c34f8a --- /dev/null +++ b/delivery.go @@ -0,0 +1,302 @@ +package imapmaildir + +import ( + "errors" + "io" + "os" + "runtime" + "strings" + "sync" + + "github.com/emersion/go-imap/backend" + "github.com/emersion/go-message/textproto" +) + +type DeliveryTarget interface { + NewDelivery() Delivery +} + +type Delivery interface { + AddRcpt(username string, userHeader textproto.Header) error + Mailbox(name string) error + SpecialMailbox(attribute, fallbackName string) error + UserMailbox(username, mailbox string, flags []string) + BodyRaw(message io.Reader) error + BodyParsed(header textproto.Header, bodyLen int, body Buffer) error + Abort() error + Commit() error +} + +type Buffer interface { + Open() (io.ReadCloser, error) +} + +type delivery struct { + b *Backend + + recipients map[string]*deliveryRecipient + defaultMailbox string + spoolPath string +} + +type deliveryRecipient struct { + username string + mailbox string + flags []string +} + +func (b *Backend) NewDelivery() Delivery { + return &delivery{ + b: b, + recipients: map[string]*deliveryRecipient{}, + defaultMailbox: InboxName, + } +} + +func (d *delivery) AddRcpt(username string, _ textproto.Header) error { + if username == "" { + return errors.New("maildir: empty recipient") + } + if _, ok := d.recipients[username]; !ok { + d.recipients[username] = &deliveryRecipient{username: username} + } + return nil +} + +func (d *delivery) Mailbox(name string) error { + if name == "" { + name = InboxName + } + d.defaultMailbox = name + return nil +} + +func (d *delivery) SpecialMailbox(attribute, fallbackName string) error { + if attribute != "" { + specs := d.b.DefaultMailboxes + if len(specs) == 0 { + specs = defaultMailboxSpecs + } + for _, spec := range specs { + if strings.EqualFold(spec.SpecialUse, attribute) { + return d.Mailbox(spec.Name) + } + } + } + return d.Mailbox(fallbackName) +} + +func (d *delivery) UserMailbox(username, mailbox string, flags []string) { + if username == "" { + return + } + rcpt, ok := d.recipients[username] + if !ok { + rcpt = &deliveryRecipient{username: username} + d.recipients[username] = rcpt + } + rcpt.mailbox = mailbox + rcpt.flags = append([]string{}, flags...) +} + +func (d *delivery) BodyRaw(message io.Reader) error { + if message == nil { + return errors.New("maildir: missing message body") + } + return d.writeSpool(func(w io.Writer) error { + _, err := io.Copy(w, message) + return err + }) +} + +func (d *delivery) BodyParsed(header textproto.Header, _ int, body Buffer) error { + reader, err := body.Open() + if err != nil { + return err + } + defer reader.Close() + return d.writeSpool(func(w io.Writer) error { + if err := textproto.WriteHeader(w, header); err != nil { + return err + } + _, err := io.Copy(w, reader) + return err + }) +} + +func (d *delivery) Abort() error { + d.recipients = nil + d.cleanupSpool() + return nil +} + +func (d *delivery) Commit() error { + if d.recipients == nil { + return errors.New("maildir: delivery not initialized") + } + if len(d.recipients) == 0 { + return errors.New("maildir: missing recipients") + } + if d.spoolPath == "" { + return errors.New("maildir: missing message body") + } + defer d.cleanupSpool() + + defaultMailbox := d.defaultMailbox + if defaultMailbox == "" { + defaultMailbox = InboxName + } + + recipients := make([]*deliveryRecipient, 0, len(d.recipients)) + for _, rcpt := range d.recipients { + recipients = append(recipients, rcpt) + } + if len(recipients) == 0 { + return nil + } + + workers := runtime.GOMAXPROCS(0) + if workers < 1 { + workers = 1 + } + if workers > len(recipients) { + workers = len(recipients) + } + + workCh := make(chan *deliveryRecipient) + var wg sync.WaitGroup + var firstErr error + var errOnce sync.Once + done := make(chan struct{}) + setErr := func(err error) { + errOnce.Do(func() { + firstErr = err + close(done) + }) + } + + worker := func() { + defer wg.Done() + for { + select { + case <-done: + return + case rcpt, ok := <-workCh: + if !ok { + return + } + mailbox := rcpt.mailbox + if mailbox == "" { + mailbox = defaultMailbox + } + reader, err := os.Open(d.spoolPath) + if err != nil { + setErr(err) + continue + } + err = d.b.deliverToRecipient(rcpt.username, mailbox, rcpt.flags, reader) + closeErr := reader.Close() + if err == nil { + err = closeErr + } + if err != nil { + setErr(err) + } + } + } + } + + for i := 0; i < workers; i++ { + wg.Add(1) + go worker() + } +recipientLoop: + for _, rcpt := range recipients { + select { + case <-done: + break recipientLoop + case workCh <- rcpt: + } + } + close(workCh) + wg.Wait() + return firstErr +} + +func (d *delivery) writeSpool(fn func(io.Writer) error) error { + d.cleanupSpool() + file, err := os.CreateTemp("", "maildir-delivery-*") + if err != nil { + return err + } + name := file.Name() + if err := fn(file); err != nil { + _ = file.Close() + _ = os.Remove(name) + return err + } + if err := file.Close(); err != nil { + _ = os.Remove(name) + return err + } + d.spoolPath = name + return nil +} + +func (d *delivery) cleanupSpool() { + if d.spoolPath == "" { + return + } + _ = os.Remove(d.spoolPath) + d.spoolPath = "" +} + +func (b *Backend) deliverToRecipient(username, mailbox string, flags []string, reader io.Reader) error { + user, err := b.GetUser(username) + if err != nil { + return err + } + u, ok := user.(*User) + if !ok { + return errors.New("maildir: unsupported user implementation") + } + + mboxDir, err := u.storage.Dir(mailbox) + if err != nil { + return err + } + exists, err := mboxDir.Exists() + if err != nil { + return err + } + if !exists { + if strings.EqualFold(mailbox, InboxName) { + if err := mboxDir.Init(); err != nil { + return err + } + } else { + return backend.ErrNoSuchMailbox + } + } + + delivery, err := mboxDir.NewDelivery() + if err != nil { + return err + } + if _, err := io.Copy(delivery, reader); err != nil { + _ = delivery.Abort() + return err + } + if err := delivery.Close(); err != nil { + return err + } + + u.mailboxesLock.Lock() + mbox := u.ensureMailbox(mailbox) + u.mailboxesLock.Unlock() + if mbox == nil { + return errors.New("maildir: failed to load mailbox state") + } + _, err = mbox.listEntries() + return err +} diff --git a/delivery_fs_test.go b/delivery_fs_test.go new file mode 100644 index 0000000..c417444 --- /dev/null +++ b/delivery_fs_test.go @@ -0,0 +1,28 @@ +package imapmaildir + +import ( + "os" + "testing" + + "github.com/foxcpp/go-imap-maildir/maildir/fs" +) + +func TestBackendDeliveryFS(t *testing.T) { + runDeliveryTests(t, newTestBackend) +} + +func newTestBackend(t *testing.T) (*Backend, func()) { + root, err := os.MkdirTemp("", "go-imap-maildir-delivery-") + if err != nil { + t.Fatal(err) + } + backend, err := New(root+"/{username}", fs.Provider{}, nil) + if err != nil { + _ = os.RemoveAll(root) + t.Fatal(err) + } + cleanup := func() { + _ = os.RemoveAll(root) + } + return backend, cleanup +} diff --git a/delivery_s3_test.go b/delivery_s3_test.go new file mode 100644 index 0000000..08c2a6b --- /dev/null +++ b/delivery_s3_test.go @@ -0,0 +1,23 @@ +package imapmaildir + +import ( + "testing" + + "github.com/foxcpp/go-imap-maildir/maildir/s3" + "github.com/foxcpp/go-imap-maildir/maildir/s3/testing" +) + +func TestBackendDeliveryS3(t *testing.T) { + runDeliveryTests(t, newTestBackendS3) +} + +func newTestBackendS3(t *testing.T) (*Backend, func()) { + client, bucket, rootPrefix, cleanup := s3testing.StartMinio(t) + provider := s3.NewProvider(client, bucket, rootPrefix) + backend, err := New("users/{username}", provider, nil) + if err != nil { + cleanup() + t.Fatal(err) + } + return backend, cleanup +} diff --git a/delivery_test.go b/delivery_test.go new file mode 100644 index 0000000..f810238 --- /dev/null +++ b/delivery_test.go @@ -0,0 +1,263 @@ +package imapmaildir + +import ( + "bytes" + "io" + "strings" + "testing" + + "github.com/emersion/go-imap" + "github.com/emersion/go-message/textproto" + + "github.com/foxcpp/go-imap-maildir/maildir" +) + +type backendFactory func(t *testing.T) (*Backend, func()) + +func runDeliveryTests(t *testing.T, factory backendFactory) { + t.Run("commit", func(t *testing.T) { + backend, cleanup := factory(t) + defer cleanup() + + if err := backend.CreateUser("alice"); err != nil { + t.Fatal(err) + } + + delivery := backend.NewDelivery() + if err := delivery.AddRcpt("alice", textproto.Header{}); err != nil { + t.Fatal(err) + } + if err := delivery.BodyRaw(bytes.NewReader([]byte("hello"))); err != nil { + t.Fatal(err) + } + if err := delivery.Commit(); err != nil { + t.Fatal(err) + } + + msgs := mustUnseen(t, backend, "alice", InboxName) + if len(msgs) != 1 { + t.Fatalf("expected 1 message in new, got %d", len(msgs)) + } + if readMessage(t, msgs[0]) != "hello" { + t.Fatalf("expected delivered message content") + } + }) + + t.Run("abort", func(t *testing.T) { + backend, cleanup := factory(t) + defer cleanup() + + if err := backend.CreateUser("alice"); err != nil { + t.Fatal(err) + } + + delivery := backend.NewDelivery() + if err := delivery.AddRcpt("alice", textproto.Header{}); err != nil { + t.Fatal(err) + } + if err := delivery.BodyRaw(bytes.NewReader([]byte("discard"))); err != nil { + t.Fatal(err) + } + if err := delivery.Abort(); err != nil { + t.Fatal(err) + } + + msgs := mustUnseen(t, backend, "alice", InboxName) + if len(msgs) != 0 { + t.Fatalf("expected 0 messages in new, got %d", len(msgs)) + } + }) + + t.Run("missing recipients", func(t *testing.T) { + backend, cleanup := factory(t) + defer cleanup() + + delivery := backend.NewDelivery() + if err := delivery.BodyRaw(bytes.NewReader([]byte("hello"))); err != nil { + t.Fatal(err) + } + if err := delivery.Commit(); err == nil { + t.Fatalf("expected error for missing recipients") + } + }) + + t.Run("missing body", func(t *testing.T) { + backend, cleanup := factory(t) + defer cleanup() + + if err := backend.CreateUser("alice"); err != nil { + t.Fatal(err) + } + + delivery := backend.NewDelivery() + if err := delivery.AddRcpt("alice", textproto.Header{}); err != nil { + t.Fatal(err) + } + if err := delivery.Commit(); err == nil { + t.Fatalf("expected error for missing body") + } + }) + + t.Run("multiple recipients", func(t *testing.T) { + backend, cleanup := factory(t) + defer cleanup() + + for _, name := range []string{"alice", "bob"} { + if err := backend.CreateUser(name); err != nil { + t.Fatal(err) + } + } + + delivery := backend.NewDelivery() + if err := delivery.AddRcpt("alice", textproto.Header{}); err != nil { + t.Fatal(err) + } + if err := delivery.AddRcpt("bob", textproto.Header{}); err != nil { + t.Fatal(err) + } + if err := delivery.BodyRaw(bytes.NewReader([]byte("hello"))); err != nil { + t.Fatal(err) + } + if err := delivery.Commit(); err != nil { + t.Fatal(err) + } + + if len(mustUnseen(t, backend, "alice", InboxName)) != 1 { + t.Fatalf("expected alice delivery") + } + if len(mustUnseen(t, backend, "bob", InboxName)) != 1 { + t.Fatalf("expected bob delivery") + } + }) + + t.Run("mailbox override", func(t *testing.T) { + backend, cleanup := factory(t) + defer cleanup() + + if err := backend.CreateUser("alice"); err != nil { + t.Fatal(err) + } + + delivery := backend.NewDelivery() + if err := delivery.AddRcpt("alice", textproto.Header{}); err != nil { + t.Fatal(err) + } + if err := delivery.Mailbox("Archive"); err != nil { + t.Fatal(err) + } + if err := delivery.BodyRaw(bytes.NewReader([]byte("hello"))); err != nil { + t.Fatal(err) + } + if err := delivery.Commit(); err != nil { + t.Fatal(err) + } + + if len(mustUnseen(t, backend, "alice", InboxName)) != 0 { + t.Fatalf("expected inbox empty") + } + if len(mustUnseen(t, backend, "alice", "Archive")) != 1 { + t.Fatalf("expected archive delivery") + } + }) + + t.Run("special mailbox", func(t *testing.T) { + backend, cleanup := factory(t) + defer cleanup() + + if err := backend.CreateUser("alice"); err != nil { + t.Fatal(err) + } + + delivery := backend.NewDelivery() + if err := delivery.AddRcpt("alice", textproto.Header{}); err != nil { + t.Fatal(err) + } + if err := delivery.SpecialMailbox(imap.TrashAttr, "Trash"); err != nil { + t.Fatal(err) + } + if err := delivery.BodyRaw(bytes.NewReader([]byte("hello"))); err != nil { + t.Fatal(err) + } + if err := delivery.Commit(); err != nil { + t.Fatal(err) + } + + if len(mustUnseen(t, backend, "alice", InboxName)) != 0 { + t.Fatalf("expected inbox empty") + } + if len(mustUnseen(t, backend, "alice", "Trash")) != 1 { + t.Fatalf("expected trash delivery") + } + }) + + t.Run("body parsed", func(t *testing.T) { + backend, cleanup := factory(t) + defer cleanup() + + if err := backend.CreateUser("alice"); err != nil { + t.Fatal(err) + } + + delivery := backend.NewDelivery() + if err := delivery.AddRcpt("alice", textproto.Header{}); err != nil { + t.Fatal(err) + } + header := textproto.Header{} + header.Set("Subject", "Test") + body := testBuffer{r: bytes.NewReader([]byte("payload"))} + if err := delivery.BodyParsed(header, 7, body); err != nil { + t.Fatal(err) + } + if err := delivery.Commit(); err != nil { + t.Fatal(err) + } + + msgs := mustUnseen(t, backend, "alice", InboxName) + if len(msgs) != 1 { + t.Fatalf("expected 1 message in new, got %d", len(msgs)) + } + content := readMessage(t, msgs[0]) + if !strings.Contains(content, "Subject: Test") || !strings.Contains(content, "payload") { + t.Fatalf("expected header and body in delivered message") + } + }) +} + +func readMessage(t *testing.T, msg maildir.Message) string { + r, err := msg.Open() + if err != nil { + t.Fatal(err) + } + defer r.Close() + buf := &bytes.Buffer{} + if _, err := buf.ReadFrom(r); err != nil { + t.Fatal(err) + } + return strings.TrimSpace(buf.String()) +} + +func mustUnseen(t *testing.T, backend *Backend, username, mailbox string) []maildir.Message { + user, err := backend.GetUser(username) + if err != nil { + t.Fatal(err) + } + u := user.(*User) + + mbox, err := u.storage.Dir(mailbox) + if err != nil { + t.Fatal(err) + } + msgs, err := mbox.Unseen() + if err != nil { + t.Fatal(err) + } + return msgs +} + +type testBuffer struct { + r *bytes.Reader +} + +func (t testBuffer) Open() (io.ReadCloser, error) { + return io.NopCloser(t.r), nil +} diff --git a/fetch.go b/fetch.go index 2d27004..d39fbf9 100644 --- a/fetch.go +++ b/fetch.go @@ -5,108 +5,78 @@ import ( "bytes" "fmt" "io" - "os" "github.com/emersion/go-imap" "github.com/emersion/go-imap/backend/backendutil" "github.com/emersion/go-message/textproto" + + "github.com/foxcpp/go-imap-maildir/maildir" ) -func (m *Mailbox) fetch(ch chan<- *imap.Message, seqNum uint32, msg message, items []imap.FetchItem) error { +func (m *SelectedMailbox) fetch(seqNum uint32, entry msgEntry, items []imap.FetchItem, recent bool) (*imap.Message, error) { result := imap.NewMessage(seqNum, items) var ( - info messageInfo - cache messageCache - flags messageFlags - - bodyItems []imap.FetchItem - header textproto.Header + bodyItems []imap.FetchItem + header textproto.Header + infoLoaded bool + info maildir.Info ) for _, item := range items { switch item { case imap.FetchUid: - result.Uid = msg.UID + result.Uid = entry.uid case imap.FetchFlags: - if flags.UID == 0 { - if err := m.handle.One("UID", msg.UID, &flags); err != nil { - return fmt.Errorf("fetch: flags query: %w", err) - } - m.sanityCheckFlags(&msg, flags.Flags) - } - m.b.Debug.Println("fetch: loaded flags for uid", msg.UID, flags.Flags) - result.Flags = flags.Flags + result.Flags = m.entryFlags(entry, recent) case imap.FetchInternalDate: - if info.UID == 0 { - if err := m.handle.One("UID", msg.UID, &info); err != nil { - return fmt.Errorf("fetch: info query: %w", err) - } - } - m.b.Debug.Println("fetch: loaded date for uid", msg.UID, flags.Flags) - result.InternalDate = info.InternalDate - case imap.FetchRFC822Size: - if info.UID == 0 { - if err := m.handle.One("UID", msg.UID, &info); err != nil { - return fmt.Errorf("fetch: info query: %w", err) - } - } - result.Size = info.RFC822Size - case imap.FetchEnvelope: - if cache.UID == 0 { - if err := m.handle.One("UID", msg.UID, &cache); err != nil { - return fmt.Errorf("fetch: cache query: %w", err) - } - } - if cache.Envelope != nil { - result.Envelope = cache.Envelope - m.b.Debug.Println("fetch: cache hit for envelope", msg.UID, cache.Envelope) + if !entry.meta.internalDate.IsZero() { + result.InternalDate = entry.meta.internalDate continue } - bodyItems = append(bodyItems, item) - case imap.FetchBodyStructure, imap.FetchBody: - if cache.UID == 0 { - if err := m.handle.One("UID", msg.UID, &cache); err != nil { - return fmt.Errorf("fetch: cache query: %w", err) + if !infoLoaded { + var err error + info, err = entry.msg.Stat() + if err != nil { + return nil, fmt.Errorf("fetch: stat: %w", err) } + infoLoaded = true } - if cache.BodyStructure != nil { - if item == imap.FetchBody { - result.BodyStructure = stripExtBodyStruct(cache.BodyStructure) - } else { - result.BodyStructure = cache.BodyStructure + result.InternalDate = info.ModTime() + case imap.FetchRFC822Size: + if !infoLoaded { + var err error + info, err = entry.msg.Stat() + if err != nil { + return nil, fmt.Errorf("fetch: stat: %w", err) } - m.b.Debug.Println("fetch: cache hit for body structure", msg.UID, cache.BodyStructure) - continue + infoLoaded = true } + result.Size = uint32(info.Size()) + case imap.FetchEnvelope, imap.FetchBodyStructure, imap.FetchBody: bodyItems = append(bodyItems, item) default: bodyItems = append(bodyItems, item) } } - filePath, err := m.dir.Filename(msg.key()) - if err != nil { - return fmt.Errorf("fetch: filename resolve: %w", err) - } - for _, item := range bodyItems { - err := m.fetchBodyItem(result, &header, filePath, item) + err := m.fetchBodyItem(result, &header, entry.msg, item) if err != nil { - return err + return nil, err } } - ch <- result - - return nil + return result, nil } -func (m *Mailbox) fetchBodyItem(result *imap.Message, header *textproto.Header, filePath string, item imap.FetchItem) error { - // TODO: Figure out how to avoid re-opening the file each time. +func (m *SelectedMailbox) fetchBodyItem(result *imap.Message, header *textproto.Header, msg maildir.Message, item imap.FetchItem) error { openBody := func() (*bufio.Reader, io.Closer, error) { - f, err := os.Open(filePath) - return bufio.NewReader(f), f, err + f, err := msg.Open() + if err != nil { + return nil, nil, err + } + return bufio.NewReader(f), f, nil } ensureHeader := func(bufR *bufio.Reader) error { if header.Len() != 0 { @@ -168,7 +138,6 @@ func (m *Mailbox) fetchBodyItem(result *imap.Message, header *textproto.Header, literal, err := backendutil.FetchBodySection(*header, bufR, sectName) if err != nil { - m.error("body section fetch %s %v", err, filePath, sectName) literal = bytes.NewReader(nil) } result.Body[sectName] = literal @@ -179,62 +148,13 @@ func (m *Mailbox) fetchBodyItem(result *imap.Message, header *textproto.Header, func skipHeader(bufR *bufio.Reader) error { for { - // Skip header if it is not needed. line, err := bufR.ReadSlice('\n') if err != nil { return err } - // If line is empty (message uses LF delim) or contains only CR (messages uses CRLF delim) if len(line) == 0 || (len(line) == 1 || line[0] == '\r') { break } } return nil } - -func stripExtBodyStruct(extended *imap.BodyStructure) *imap.BodyStructure { - stripped := *extended - stripped.Extended = false - stripped.Disposition = "" - stripped.DispositionParams = nil - stripped.Language = nil - stripped.Location = nil - stripped.MD5 = "" - - for i := range stripped.Parts { - stripped.Parts[i] = stripExtBodyStruct(stripped.Parts[i]) - } - return &stripped -} - -func (m *Mailbox) sanityCheckFlags(msg *message, flags []string) { - var ( - hasSeen = false - hasDeleted = false - mismatch = false - ) - for _, f := range flags { - if f == imap.DeletedFlag { - hasDeleted = true - } - if f == imap.SeenFlag { - hasSeen = true - } - } - if hasSeen != !msg.Unseen { - m.error("BUG: message-messageFlags mismatch, flags: %v, message: %+v", nil, flags, msg) - mismatch = true - } - if hasDeleted != msg.Deleted { - m.error("BUG: message-messageFlags mismatch, flags: %v, message: %+v", nil, flags, msg) - mismatch = true - } - - if mismatch { - msg.Deleted = hasDeleted - msg.Unseen = !hasSeen - if err := m.handle.Save(msg); err != nil { - m.error("I/O error", err) - } - } -} diff --git a/go.mod b/go.mod index 041cfad..01a398f 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,39 @@ module github.com/foxcpp/go-imap-maildir -go 1.13 +go 1.24.0 require ( - github.com/asdine/storm v2.1.2+incompatible // indirect - github.com/asdine/storm/v3 v3.1.0 - github.com/emersion/go-imap v1.0.4-0.20200128190657-5162c2f0c9e1 - github.com/emersion/go-maildir v0.2.0 - github.com/emersion/go-message v0.11.2 - github.com/foxcpp/go-imap-backend-tests v0.0.0-20200616221226-85255dc9f40f - go.etcd.io/bbolt v1.3.3 - golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect + github.com/dgraph-io/ristretto/v2 v2.4.0 + github.com/emersion/go-imap v1.2.2-0.20220928192137-6fac715be9cf + github.com/emersion/go-maildir v0.6.0 + github.com/emersion/go-message v0.18.2 + github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16 + github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613 + github.com/minio/minio-go/v7 v7.0.98 ) -replace github.com/foxcpp/go-imap-backend-tests => ../go-imap-backend-tests/ +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + gotest.tools v2.2.0+incompatible // indirect +) + +replace github.com/emersion/go-imap => github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887 diff --git a/go.sum b/go.sum index 251838f..236cef5 100644 --- a/go.sum +++ b/go.sum @@ -1,68 +1,111 @@ -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM= -github.com/asdine/storm v1.1.0 h1:lwDLqMMPhokfYk8EuU1RRHTi54T68EI+QnCqK5t4TCM= -github.com/asdine/storm v2.1.2+incompatible h1:dczuIkyqwY2LrtXPz8ixMrU/OFgZp71kbKTHGrXYt/Q= -github.com/asdine/storm v2.1.2+incompatible/go.mod h1:RarYDc9hq1UPLImuiXK3BIWPJLdIygvV3PsInK0FbVQ= -github.com/asdine/storm/v3 v3.1.0 h1:yrpSNS+E7ef5Y5KjyZDeyW72Dl17lYG7oZ7eUoWvo5s= -github.com/asdine/storm/v3 v3.1.0/go.mod h1:letAoLCXz4UfodwNgMNILMb2oRH+su337ZfHnkRzqDA= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emersion/go-imap v1.0.0-beta.4.0.20190504114255-4d5af3d05147/go.mod h1:mOPegfAgLVXbhRm1bh2JTX08z2Y3HYmKYpbrKDeAzsQ= -github.com/emersion/go-imap v1.0.4-0.20200128190657-5162c2f0c9e1 h1:tWRlZJ61q8Qg47ETbkwB4fUrHtuaXhQLzf8q9XIFoDQ= -github.com/emersion/go-imap v1.0.4-0.20200128190657-5162c2f0c9e1/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU= -github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a h1:bMdSPm6sssuOFpIaveu3XGAijMS3Tq2S3EqFZmZxidc= +github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU= +github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a/go.mod h1:ikgISoP7pRAolqsVP64yMteJa2FIpS6ju88eBT6K1yQ= -github.com/emersion/go-imap-move v0.0.0-20180601155324-5eb20cb834bf h1:TmRfuPmhrwAhWKu2XaBaY9N+anRRDBO+E8VRVO9g3fY= github.com/emersion/go-imap-move v0.0.0-20180601155324-5eb20cb834bf/go.mod h1:QuMaZcKFDVI0yCrnAbPLfbwllz1wtOrZH8/vZ5yzp4w= -github.com/emersion/go-maildir v0.2.0 h1:fC4+UVGl8GcQGbFF7AWab2JMf4VbKz+bMNv07xxhzs8= -github.com/emersion/go-maildir v0.2.0/go.mod h1:I2j27lND/SRLgxROe50Vam81MSaqPFvJ0OHNnDZ7n84= -github.com/emersion/go-message v0.9.1/go.mod h1:m3cK90skCWxm5sIMs1sXxly4Tn9Plvcf6eayHZJ1NzM= -github.com/emersion/go-message v0.10.3/go.mod h1:3h+HsGTCFHmk4ngJ2IV/YPhdlaOcR6hcgqM3yca9v7c= -github.com/emersion/go-message v0.11.1 h1:0C/S4JIXDTSfXB1vpqdimAYyK4+79fgEAMQ0dSL+Kac= -github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY= -github.com/emersion/go-message v0.11.2 h1:oxO9SQ+3wgBAQRdk07eqfkCJ26Tl8ZHF7CcpGVoE00o= -github.com/emersion/go-message v0.11.2/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY= -github.com/emersion/go-sasl v0.0.0-20161116183048-7e096a0a6197/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= -github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b h1:uhWtEWBHgop1rqEk2klKaxPAkVDCXexai6hSuRQ7Nvs= -github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= -github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe h1:40SWqY0zE3qCi6ZrtTf5OUdNm5lDnGnjRSq9GgmeTrg= -github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= -github.com/foxcpp/go-imap-backend-tests v0.0.0-20200104150404-bd7815ad9f5a h1:0M1igChqRDhGcWrjuAq+BiYC7CdVCsyktmbKWlJeuuo= -github.com/foxcpp/go-imap-backend-tests v0.0.0-20200104150404-bd7815ad9f5a/go.mod h1:yUISYv/uXLQ6tQZcds/p/hdcZ5JzrEUifyED2VffWpc= -github.com/foxcpp/go-imap-backend-tests v0.0.0-20200616221226-85255dc9f40f h1:vKaWiNDR2dR6qyF/9zjot5Q3ghOEw9Fmj0ltYvBkCwM= -github.com/foxcpp/go-imap-backend-tests v0.0.0-20200616221226-85255dc9f40f/go.mod h1:yUISYv/uXLQ6tQZcds/p/hdcZ5JzrEUifyED2VffWpc= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/emersion/go-maildir v0.6.0 h1:MPx2RSS1Xq8j1cNOzfq7YyF+5Leoeif1XqSeuytdET8= +github.com/emersion/go-maildir v0.6.0/go.mod h1:Wpgtt9EOIJWe++WKa+JRvDwv+qIV7MeFdvZu/VbsXN4= +github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4= +github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= +github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= +github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887 h1:qUoaaHyrRpQw85ru6VQcC6JowdhrWl7lSbI1zRX1FTM= +github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY= +github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16 h1:qheFPDpteiUy7Ym18R68OYenpk85UyKYGkhYTmddSBg= +github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16/go.mod h1:OPP1AgKxMPo3aHX5pcEZLQhhh5sllFcB8aUN9f6a6X8= +github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613 h1:fw9OWfPxP1CK4D+XAEEg0JzhvFGo04L+F5Xw55t9s3E= +github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613/go.mod h1:P/O/qz4gaVkefzJ40BUtN/ZzBnaEg0YYe1no/SMp7Aw= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/martinlindhe/base36 v0.0.0-20190418230009-7c6542dfbb41/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= -github.com/martinlindhe/base36 v1.0.0 h1:eYsumTah144C0A8P1T/AVSUk5ZoLnhfYFM3OGQxB52A= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20191105142833-ac3223d80179/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= diff --git a/mailbox.go b/mailbox.go index 7f8e592..1ca616a 100644 --- a/mailbox.go +++ b/mailbox.go @@ -1,80 +1,53 @@ package imapmaildir import ( + "bytes" "errors" "fmt" "io" - "os" - "path/filepath" - "strconv" + "slices" + "sort" "strings" + "sync" "time" - "github.com/asdine/storm/v3" "github.com/emersion/go-imap" + "github.com/emersion/go-imap/backend" "github.com/emersion/go-imap/backend/backendutil" - "github.com/emersion/go-maildir" - "go.etcd.io/bbolt" + "github.com/emersion/go-message" + mess "github.com/foxcpp/go-imap-mess" + + "github.com/foxcpp/go-imap-maildir/maildir" ) type Mailbox struct { b *Backend - // DB handle, see dbs comment in Backend - it may be not only ours handle. - // Also it is nil for Mailbox'es created by ListMailboxes. - handle *storm.DB - dir maildir.Dir + user *User - username string + dir maildir.Dir - // Full mailbox name. name string - // Filesystem path to the mailbox directory. Not guaranteed to be absolute - // or relative. - path string -} - -type mboxData struct { - Dummy int `storm:"id"` - - UidValidity uint32 - UidNext uint32 - MsgsCount uint32 -} - -type message struct { - UID uint32 `storm:"id,increment"` - - // This structure contains minimal information about message - // because it is often range-scanned by various operations. - - // XXX Struct might get overriden by flag-related operations - // update these to not do so if more fields are added there. - - Unseen bool - Deleted bool -} + state *mailboxState + subscribed bool -func (m message) key() string { - return "imap-" + strconv.FormatUint(uint64(m.UID), 10) + limitLock sync.Mutex + appendLimit *uint32 } -type messageFlags struct { - UID uint32 `storm:"id,increment"` - Flags []string +type SelectedMailbox struct { + *Mailbox + conn backend.Conn + readOnly bool + handle *mess.MailboxHandle } -type messageInfo struct { - UID uint32 `storm:"id,increment"` - RFC822Size uint32 - InternalDate time.Time -} - -type messageCache struct { - UID uint32 `storm:"id"` - Envelope *imap.Envelope - BodyStructure *imap.BodyStructure +type msgEntry struct { + msg maildir.Message + meta *messageMeta + uid uint32 + seqNum uint32 } func (m *Mailbox) Name() string { @@ -82,615 +55,761 @@ func (m *Mailbox) Name() string { } func (m *Mailbox) Info() (*imap.MailboxInfo, error) { - // TODO: CHILDREN extension - // TODO: SPECIAL-USE extension + info := &imap.MailboxInfo{ + Delimiter: HierarchySep, + Name: m.name, + } - // This function should complete without using DB as it will be called - // by LIST handler in go-imap and ListMailboxes does not initialize it for - // performance reasons. + if strings.Count(m.name, HierarchySep) == MaxMboxNesting { + info.Attributes = append(info.Attributes, imap.NoInferiorsAttr) + } - info := &imap.MailboxInfo{ - Attributes: nil, - Delimiter: HierarchySep, - Name: m.name, + hasChildren := false + childDir := m.dir + if strings.EqualFold(m.name, InboxName) { + inboxDir, err := m.user.storage.Dir(InboxName) + if err != nil { + return nil, fmt.Errorf("I/O error: %w", err) + } + childDir = inboxDir } - _, err := os.Stat(filepath.Join(m.path, "cur")) + children, err := childDir.Children() if err != nil { - if os.IsNotExist(err) { + if maildir.IsNotExist(err) { info.Attributes = append(info.Attributes, imap.NoSelectAttr) } else { - return nil, errors.New("I/O error") + return nil, fmt.Errorf("I/O error: %w", err) } + } else if len(children) > 0 { + hasChildren = true } - if strings.Count(m.name, HierarchySep) == MaxMboxNesting { - info.Attributes = append(info.Attributes, imap.NoInferiorsAttr) + if hasChildren { + info.Attributes = append(info.Attributes, imap.HasChildrenAttr) + } else { + info.Attributes = append(info.Attributes, imap.HasNoChildrenAttr) + } + + if specialUses, err := m.user.mailboxSpecialUse(m); err == nil { + for _, use := range specialUses { + info.Attributes = append(info.Attributes, use) + } } return info, nil } -func (m *Mailbox) error(descr string, cause error, args ...interface{}) { - if cause == nil { - m.b.Log.Printf("mailbox %s %s: %s", m.username, m.name, fmt.Sprintf(descr, args...)) - } else { - m.b.Log.Printf("mailbox %s %s: %s: %v", m.username, m.name, fmt.Sprintf(descr, args...), cause) - } +func (m *SelectedMailbox) Conn() backend.Conn { + return m.conn } -func (m *Mailbox) Status(items []imap.StatusItem) (*imap.MailboxStatus, error) { - status := imap.NewMailboxStatus(m.name, items) +func (m *SelectedMailbox) Poll(expunge bool) error { + m.handle.Sync(expunge) + return nil +} - status.Flags = []string{ - imap.SeenFlag, imap.AnsweredFlag, imap.FlaggedFlag, - imap.DeletedFlag, imap.DraftFlag, - } - status.PermanentFlags = []string{ - imap.SeenFlag, imap.AnsweredFlag, imap.FlaggedFlag, - imap.DeletedFlag, imap.DraftFlag, - } - // TODO: Report used flags (cache them?) +func (m *SelectedMailbox) Check() error { + return nil +} - var mboxMeta mboxData +func (m *SelectedMailbox) Status(items []imap.StatusItem) (*imap.MailboxStatus, error) { + return m.user.Status(m.name, items) +} - if err := m.handle.One("Dummy", 1, &mboxMeta); err != nil { - m.error("Status: fetch mboxData", err) - return nil, errors.New("I/O error") - } +func (m *SelectedMailbox) SetSubscribed(subscribed bool) error { + return m.user.SetSubscribed(m.name, subscribed) +} - var ( - needMsgCount bool - needRecent bool - needUnseen bool - msgCounter uint32 - ) - for _, item := range items { - switch item { - case imap.StatusMessages: - needMsgCount = true - case imap.StatusRecent: - // TODO: Consider using "new" directory for implementing Recent - needRecent = true - case imap.StatusUidNext: - status.UidNext = mboxMeta.UidNext - case imap.StatusUidValidity: - status.UidValidity = mboxMeta.UidValidity - case imap.StatusUnseen: - needUnseen = true - default: - return nil, fmt.Errorf("unknown status item: %s", item) - } - } +func (m *SelectedMailbox) CreateMessage(flags []string, date time.Time, body imap.Literal) error { + return m.user.CreateMessage(m.name, flags, date, body, nil) +} - q := m.handle.Select().OrderBy("UID") - err := q.Each(new(message), func(rec interface{}) error { - msg := rec.(*message) - msgCounter++ - if needUnseen && msg.Unseen { - status.Unseen++ - } - if status.UnseenSeqNum == 0 && msg.Unseen { - status.UnseenSeqNum = uint32(msgCounter + 1) - } +func (m *SelectedMailbox) Idle(done <-chan struct{}) { + m.handle.Idle(done) +} + +func (m *SelectedMailbox) Close() error { + return m.handle.Close() +} + +func (m *Mailbox) CreateMessageLimit() *uint32 { + m.limitLock.Lock() + defer m.limitLock.Unlock() + + if m.appendLimit == nil { return nil - }) - if err != nil && err != storm.ErrNotFound { - m.error("Status", err) - return nil, errors.New("I/O error") } - if needMsgCount { - status.Messages = msgCounter + val := *m.appendLimit + return &val +} + +func (m *Mailbox) updateUidlistFilename(uid uint32, msg maildir.Message) { + if m.state.meta == nil { + return } - if needRecent { - status.Recent = msgCounter + filename := msg.Name() + if m.state.meta.FilenameByUID[uid] != filename { + m.state.meta.FilenameByUID[uid] = filename + m.state.meta.DirtyUIDList = true } +} - if needMsgCount && msgCounter != mboxMeta.MsgsCount { - m.b.Log.Printf("mailbox %s/%s: BUG: cached message count de-sync, actual: %d, cache: %d", - m.username, m.name, msgCounter, mboxMeta.MsgsCount) +func (m *Mailbox) SetMessageLimit(val *uint32) error { + m.limitLock.Lock() + defer m.limitLock.Unlock() - mboxMeta.MsgsCount = msgCounter - if err := m.handle.Set("Dummy", 1, &mboxMeta); err != nil { - m.error("Status: fix-up message count", err) - } + if val == nil { + m.appendLimit = nil + return nil } + copyVal := *val + m.appendLimit = ©Val + return nil +} - return status, nil +func (m *SelectedMailbox) CreateMessageLimit() *uint32 { + return m.Mailbox.CreateMessageLimit() } -func (m *Mailbox) SetSubscribed(subscribed bool) error { - // No-op. Subscription status is not implemented. +func (m *SelectedMailbox) SetMessageLimit(val *uint32) error { + if err := m.Mailbox.SetMessageLimit(val); err != nil { + return err + } + m.user.setMailboxLimit(m.name, val) return nil } -func (m *Mailbox) Check() error { - // No Check necessary as changes are coordinated via single Backend instance. - return nil +func (m *Mailbox) mailboxKey() string { + return m.user.name + "\x00" + m.name } -func (m *Mailbox) contains(seq *imap.SeqSet, id uint32, last bool) bool { - if seq.Contains(id) { - return true +func (m *Mailbox) uidValidity() uint32 { + _ = m.loadMetadataState() + m.b.statesLock.Lock() + defer m.b.statesLock.Unlock() + + if m.state.meta != nil && m.state.meta.UIDValidity != 0 { + return m.state.meta.UIDValidity } + if m.state.uidValidity == 0 { + m.state.uidValidity = 1 + } + return m.state.uidValidity +} - if seq.Dynamic() { - for _, s := range seq.Set { - if s.Stop == 0 && last { - return true - } +func (m *Mailbox) listEntries() ([]msgEntry, error) { + recentMsgs, err := m.dir.Unseen() + if err != nil { + if !maildir.IsNotExist(err) { + return nil, err } + recentMsgs = nil + } + recentKeys := map[string]struct{}{} + for _, msg := range recentMsgs { + recentKeys[msg.Key()] = struct{}{} } - return false -} -func (m *Mailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []imap.FetchItem, ch chan<- *imap.Message) error { - defer close(ch) + if err := m.loadMetadataState(); err != nil { + return nil, err + } - var mboxMeta mboxData - if err := m.handle.One("Dummy", 1, &mboxMeta); err != nil { - m.error("list: I/O error for mboxMeta", err) - return errors.New("I/O error, try again later") + msgs, err := m.dir.Messages() + if err != nil { + return nil, err } - errored := false + sort.Slice(msgs, func(i, j int) bool { + return msgs[i].Key() < msgs[j].Key() + }) - q := m.handle.Select().OrderBy("UID") + present := make(map[string]bool, len(msgs)) + entries := make([]msgEntry, 0, len(msgs)) - maxSeqNum, err := q.Count(new(message)) - if err != nil { - m.error("list: I/O error for mboxMeta", err) - return errors.New("I/O error, try again later") - } + m.b.statesLock.Lock() + defer m.b.statesLock.Unlock() - var seqNum uint32 - err = q.Each(new(message), func(rec interface{}) error { - msg := rec.(*message) - seqNum++ + state := m.state.meta + for _, msg := range msgs { + key := msg.Key() + present[key] = true - if uid && !m.contains(seqset, msg.UID, seqNum == uint32(maxSeqNum)) { - return nil + meta, ok := m.state.messages[key] + if !ok { + meta = &messageMeta{} } - if !uid && !m.contains(seqset, seqNum, seqNum == uint32(maxSeqNum)) { - return nil + + internalDate := meta.internalDate + if internalDate.IsZero() { + if info, err := msg.Stat(); err == nil { + internalDate = info.ModTime() + } } - m.b.Debug.Println("ListMessages: fetching", items, "for", seqNum, msg.UID) - if err := m.fetch(ch, seqNum, *msg, items); err != nil { - m.error("fetch", err) - errored = true - return nil + uid := state.UIDByKey[key] + if uid == 0 { + uid = state.UIDNext + state.UIDNext++ + state.UIDByKey[key] = uid + state.DirtyUIDList = true + storeRecent := m.b.Manager.NewMessage(m.mailboxKey(), uid) + meta.recent = storeRecent } - return nil - }) - if err != nil { - if err == storm.ErrNotFound { - if uid { - return nil - } - return errors.New("No messages") + meta.uid = uid + meta.flags = m.imapFlagsFromMaildir(msg.Flags()) + meta.internalDate = internalDate + if _, ok := recentKeys[key]; ok { + meta.recent = true } - m.error("I/O error", err) - return err + m.state.messages[key] = meta + + if _, ok := recentKeys[key]; ok { + meta.recent = true + } + + filename := msg.Name() + if state.FilenameByUID[meta.uid] != filename { + state.FilenameByUID[meta.uid] = filename + state.DirtyUIDList = true + } + + entries = append(entries, msgEntry{msg: msg, meta: meta, uid: meta.uid}) } - if errored { - return errors.New("Server-side error occured, partial results returned") + for key := range m.state.messages { + if !present[key] { + delete(m.state.messages, key) + } } - return nil -} -func (m *Mailbox) SearchMessages(uid bool, criteria *imap.SearchCriteria) ([]uint32, error) { - return nil, errors.New("imapmaildir: not implemented") -} + m.state.uidNext = state.UIDNext + m.state.uidValidity = state.UIDValidity -func (m *Mailbox) temporaryMsgPath() string { - ts := strconv.FormatInt(time.Now().UnixNano(), 16) + sort.Slice(entries, func(i, j int) bool { + return entries[i].uid < entries[j].uid + }) + for i := range entries { + entries[i].seqNum = uint32(i + 1) + } - return filepath.Join(m.path, "tmp", ts+":2,") + if err := m.writeMetadataState(state); err != nil { + return nil, err + } + + return entries, nil } -func (m *Mailbox) CreateMessage(flags []string, date time.Time, body imap.Literal) error { - hasRecent := false - for _, f := range flags { - if f == imap.RecentFlag { - hasRecent = true +func (m *Mailbox) imapFlagsFromMaildir(flags []maildir.Flag) []string { + _ = m.loadMetadataState() + state := m.state.meta + if state == nil { + state = &maildir.Metadata{} + state.Ensure() + } + + imapFlags := make([]string, 0, len(flags)) + for _, flag := range flags { + switch flag { + case maildir.FlagSeen: + imapFlags = append(imapFlags, imap.SeenFlag) + case maildir.FlagReplied: + imapFlags = append(imapFlags, imap.AnsweredFlag) + case maildir.FlagFlagged: + imapFlags = append(imapFlags, imap.FlaggedFlag) + case maildir.FlagTrashed: + imapFlags = append(imapFlags, imap.DeletedFlag) + case maildir.FlagDraft: + imapFlags = append(imapFlags, imap.DraftFlag) + default: + if name, ok := state.NameByKeyword[rune(flag)]; ok { + imapFlags = append(imapFlags, name) + } } } - if !hasRecent { - flags = append(flags, imap.RecentFlag) + return imapFlags +} + +func (m *Mailbox) maildirFlagsFromImap(flags []string) []maildir.Flag { + _ = m.loadMetadataState() + state := m.state.meta + if state == nil { + state = &maildir.Metadata{} + state.Ensure() + } + + seen := false + replied := false + flagged := false + trashed := false + draft := false + keywords := make([]rune, 0, len(flags)) + + for _, flag := range flags { + switch flag { + case imap.SeenFlag: + seen = true + case imap.AnsweredFlag: + replied = true + case imap.FlaggedFlag: + flagged = true + case imap.DeletedFlag: + trashed = true + case imap.DraftFlag: + draft = true + case imap.RecentFlag: + continue + default: + if letter := state.EnsureKeyword(flag); letter != 0 { + keywords = append(keywords, letter) + } + } } - // Save message outside of transaction to reduce locking contention. - tmpPath := m.temporaryMsgPath() - // Files are read-only to help enforce the IMAP immutability requirement. - f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0440) - if err != nil { - m.error("CreateMessage", err) - return errors.New("I/O error, try again later") + result := make([]maildir.Flag, 0, 5) + if seen { + result = append(result, maildir.FlagSeen) + } + if replied { + result = append(result, maildir.FlagReplied) } - defer f.Close() + if flagged { + result = append(result, maildir.FlagFlagged) + } + if trashed { + result = append(result, maildir.FlagTrashed) + } + if draft { + result = append(result, maildir.FlagDraft) + } + for _, letter := range keywords { + result = append(result, maildir.Flag(letter)) + } + return result +} - delTemp := func() { - if err := os.Remove(tmpPath); err != nil { - m.error("CreateMessage: rollback failed: temp del", err) +func hasFlag(flags []string, flag string) bool { + return slices.Contains(flags, flag) +} + +func uniqueFlags(flags []string) []string { + seen := map[string]struct{}{} + result := make([]string, 0, len(flags)) + for _, flag := range flags { + if _, ok := seen[flag]; ok { + continue } + seen[flag] = struct{}{} + result = append(result, flag) + } + return result +} + +func (m *Mailbox) entryFlags(entry msgEntry, recent bool) []string { + flags := entry.meta.flags + if recent { + flags = append(flags, imap.RecentFlag) } + return uniqueFlags(flags) +} - // If implemented naively - this might change after io.Copy - rfc822Size := body.Len() +func (m *SelectedMailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []imap.FetchItem, ch chan<- *imap.Message) error { + defer close(ch) + defer m.handle.Sync(false) - if _, err := io.Copy(f, body); err != nil { - m.error("CreateMessage: copy", err) - delTemp() + entries, err := m.listEntries() + if err != nil { return errors.New("I/O error, try again later") } - if err := f.Sync(); err != nil { - m.error("CreateMessage: copy", err) - delTemp() - return errors.New("I/O error, try again later") + shouldSetSeen := false + for _, item := range items { + sect, err := imap.ParseBodySectionName(item) + if err != nil { + continue + } + if !sect.Peek { + shouldSetSeen = true + } } - msg := message{ - Unseen: true, + itemsToFetch := items + if shouldSetSeen && !containsFetchItem(items, imap.FetchFlags) { + itemsToFetch = append(append([]imap.FetchItem{}, items...), imap.FetchFlags) } - var info messageInfo - err = m.handle.Bolt.Update(func(btx *bbolt.Tx) error { - tx := m.handle.WithTransaction(btx) - if err := tx.Save(&msg); err != nil { - return fmt.Errorf("CreateMesage: inital save: %w", err) + seqset, err = m.handle.ResolveSeq(uid, seqset) + if err != nil { + if uid { + return nil } + return err + } - info = messageInfo{ - UID: msg.UID, // magically appears there after Save - RFC822Size: uint32(rfc822Size), - InternalDate: date, - } - if err := tx.Save(&info); err != nil { - return fmt.Errorf("CreateMessage %d: info save: %w", msg.UID, err) - } - mFlags := messageFlags{ - UID: msg.UID, - Flags: flags, - } - if err := tx.Save(&mFlags); err != nil { - return fmt.Errorf("CreateMessage %d: flags save: %w", msg.UID, err) - } - cache := messageCache{UID: msg.UID} - if err := tx.Save(&cache); err != nil { - return fmt.Errorf("CreateMessage %d: cache save: %w", msg.UID, err) + for _, entry := range entries { + if !seqset.Contains(entry.uid) { + continue } - // XXX: Ignore "new" folder for now as it requires more complicated handling. - if err := os.Rename(tmpPath, filepath.Join(m.path, "cur", msg.key()+":2,")); err != nil { - return fmt.Errorf("CreateMessage %d: %w", msg.UID, err) + seq, ok := m.handle.UidAsSeq(entry.uid) + if !ok { + continue } - var mboxMeta mboxData - if err := tx.One("Dummy", 1, &mboxMeta); err != nil { - return fmt.Errorf("CreateMessage %d: load mboxData: %w", msg.UID, err) - } - mboxMeta.MsgsCount++ - // TODO: Emit StatusUpdate (or was it MailboxUpdate?). - if err := tx.Save(&mboxMeta); err != nil { - // TODO: Perhaps this should not fail and we can get away by having Status fix it? - return fmt.Errorf("CreateMessage %d: save mboxData: %w", msg.UID, err) + if shouldSetSeen && !hasFlag(entry.meta.flags, imap.SeenFlag) { + m.b.statesLock.Lock() + entry.meta.flags = uniqueFlags(append(entry.meta.flags, imap.SeenFlag)) + m.b.statesLock.Unlock() + if err := entry.msg.SetFlags(m.maildirFlagsFromImap(entry.meta.flags)); err == nil { + m.handle.FlagsChanged(entry.uid, entry.meta.flags, false) + } } - m.b.Debug.Printf("CreateMessage: written UID %d as maildir key %s to mbox %s/%s", msg.UID, msg.key(), m.username, m.name) + msg, err := m.fetch(seq, entry, itemsToFetch, m.handle.IsRecent(entry.uid)) + if err != nil { + continue + } - return nil - }) - if err != nil { - m.error("CreateMessage %d: update", err, msg.UID) - delTemp() - return errors.New("I/O error, try again later") + ch <- msg } return nil } -func (m *Mailbox) UpdateMessagesFlags(uid bool, seqset *imap.SeqSet, operation imap.FlagsOp, flags []string) error { - var ( - mFlags messageFlags - errored = false - ) - - err := m.handle.Bolt.Update(func(btx *bbolt.Tx) error { - tx := m.handle.WithTransaction(btx) +func (m *SelectedMailbox) SearchMessages(uid bool, criteria *imap.SearchCriteria) ([]uint32, error) { + entries, err := m.listEntries() + if err != nil { + return nil, errors.New("I/O error, try again later") + } - q := tx.Select().OrderBy("UID") + m.handle.ResolveCriteria(criteria) + defer m.handle.Sync(uid) - maxSeqNum, err := q.Count(new(message)) - if err != nil { - m.error("UpdateMessagesFlags: count", err) - return err + var ids []uint32 + for _, entry := range entries { + seq, ok := m.handle.UidAsSeq(entry.uid) + if !ok { + continue } - // TODO: Avoid iterating all messages for UID queries. - var seqNum uint32 - return q.Each(new(message), func(rec interface{}) error { - msg := rec.(*message) - seqNum++ - - if uid && !m.contains(seqset, msg.UID, seqNum == uint32(maxSeqNum)) { - return nil - } - if !uid && !m.contains(seqset, seqNum, seqNum == uint32(maxSeqNum)) { - return nil - } - - if err := tx.One("UID", msg.UID, &mFlags); err != nil { - m.error("UpdateMessagesFlags: fetch", err) - errored = true - return nil + entity, entErr := m.messageEntity(entry.msg) + if entity == nil { + if entErr != nil { + continue } + continue + } - m.b.Debug.Println("UpdateMessageFlags: updating flags", seqNum, msg.UID, mFlags.Flags, "op", operation, flags) - mFlags.Flags = backendutil.UpdateFlags(mFlags.Flags, operation, flags) - if err := tx.Save(&mFlags); err != nil { - m.error("UpdateMessagesFlags: save", err) - errored = true - return nil - } + flags := m.entryFlags(entry, m.handle.IsRecent(entry.uid)) + ok, err = backendutil.Match(entity, seq, entry.uid, entry.meta.internalDate, flags, criteria) + if err != nil || !ok { + continue + } - return nil - }) - }) - if err != nil { - if err == storm.ErrNotFound { - if uid { - return nil - } - return errors.New("No messages") + if uid { + ids = append(ids, entry.uid) + } else { + ids = append(ids, seq) } - m.error("I/O error", err) - return err } - if errored { - return errors.New("Server-side occured, only some messages affected") - } - return nil + return ids, nil } -func (m *Mailbox) CopyMessages(uid bool, seqset *imap.SeqSet, dest string) error { - u, err := m.b.GetUser(m.username) - if err != nil { - m.error("", err) - return err - } - tgtMboxI, err := u.GetMailbox(dest) - if err != nil { - return err +func (m *SelectedMailbox) UpdateMessagesFlags(uid bool, seqset *imap.SeqSet, operation imap.FlagsOp, silent bool, flags []string) error { + if err := m.loadMetadataState(); err != nil { + return errors.New("I/O error, try again later") } - tgtMbox := tgtMboxI.(*Mailbox) + state := m.state.meta - wrtTx, err := tgtMbox.handle.Bolt.Begin(true) - if err != nil { - return errors.New("I/O error, try again later") + newFlags := flags[:0] + for _, flag := range flags { + if flag == imap.RecentFlag { + continue + } + newFlags = append(newFlags, flag) } - txTgt := tgtMbox.handle.WithTransaction(wrtTx) - defer txTgt.Rollback() + flags = newFlags - srcTx, err := m.handle.Bolt.Begin(true) + entries, err := m.listEntries() if err != nil { return errors.New("I/O error, try again later") } - txSrc := m.handle.WithTransaction(srcTx) - defer txSrc.Rollback() - // Files that should be removed on transaction error. - var purgeList []string - rollbackFiles := func() { - for _, f := range purgeList { - if err := os.Remove(f); err != nil { - m.error("purgeList %s", err, tgtMbox.Name) - } - } - } + defer m.handle.Sync(uid) - // TODO: Avoid iterating all messages for UID queries. - q := txSrc.Select().OrderBy("UID") - - maxSeqNum, err := q.Count(new(message)) + seqset, err = m.handle.ResolveSeq(uid, seqset) if err != nil { - m.error("UpdateMessagesFlags: count", err) + if uid { + return nil + } return err } - var seqNum uint32 - err = q.Each(new(message), func(rec interface{}) error { - msg := rec.(*message) - seqNum++ - // srcName already includes directory. - srcName, err := m.dir.Filename(msg.key()) - if err != nil { - if _, ok := err.(*maildir.KeyError); ok || os.IsNotExist(err) { - m.error("CopyMessages %s: BUG: message meta-data exists but file does not", err, msg.UID) - txTgt.DeleteStruct(&msg) - txTgt.DeleteStruct(&messageInfo{UID: msg.UID}) - txTgt.DeleteStruct(&messageCache{UID: msg.UID}) - // Silently skip the message as if it was not here. - return nil - } - return fmt.Errorf("CopyMessages %d (src UID): filename error: %w", msg.UID, err) + for _, entry := range entries { + if !seqset.Contains(entry.uid) { + continue } - if uid && !m.contains(seqset, msg.UID, seqNum == uint32(maxSeqNum)) { - return nil - } - if !uid && !m.contains(seqset, seqNum, seqNum == uint32(maxSeqNum)) { - return nil - } + m.b.statesLock.Lock() + entry.meta.flags = uniqueFlags(backendutil.UpdateFlags(entry.meta.flags, operation, flags)) + m.b.statesLock.Unlock() - var info messageInfo - if err := txSrc.One("UID", msg.UID, &info); err != nil { - return fmt.Errorf("CopyMessages %d (src UID): info load: %w", msg.UID, err) - } - var cache messageCache - if err := txSrc.One("UID", msg.UID, &cache); err != nil { - return fmt.Errorf("CopyMessages %d (src UID): cache load: %w", msg.UID, err) - } - var flags messageFlags - if err := txSrc.One("UID", msg.UID, &flags); err != nil { - return fmt.Errorf("CopyMessages %d (src UID): flags load: %w", msg.UID, err) + if err := entry.msg.SetFlags(m.maildirFlagsFromImap(entry.meta.flags)); err != nil { + continue } + m.updateUidlistFilename(entry.uid, entry.msg) + m.handle.FlagsChanged(entry.uid, entry.meta.flags, silent) + } - m.b.Debug.Printf("CopyMessages: copying %d from %s", info.UID, m.name) + if err := m.writeMetadataState(state); err != nil { + return errors.New("I/O error, try again later") + } - msg.UID = 0 + return nil +} - if err := txTgt.Save(msg); err != nil { - return fmt.Errorf("CopyMessages %s, %d (src UID): initial save: %w", tgtMbox.name, info.UID, err) +func (m *SelectedMailbox) CopyMessages(uid bool, seqset *imap.SeqSet, dest string) error { + destMbox, ok := m.user.getMailbox(dest) + if !ok { + if err := m.user.validateMboxName(dest); err != nil { + return err + } + mboxDir, err := m.user.storage.Dir(dest) + if err != nil { + return err + } + exists, err := mboxDir.Exists() + if err != nil { + return errors.New("I/O error, try again later") } + if !exists { + return backend.ErrNoSuchMailbox + } + m.user.mailboxesLock.Lock() + destMbox = m.user.ensureMailbox(dest) + m.user.mailboxesLock.Unlock() + } + if err := destMbox.loadMetadataState(); err != nil { + return errors.New("I/O error, try again later") + } - m.b.Debug.Printf("CopyMessages: ... as %d to %s", msg.UID, tgtMbox.name) + entries, err := m.listEntries() + if err != nil { + return errors.New("I/O error, try again later") + } - tgtName := filepath.Join(tgtMbox.path, "cur", msg.key()+":2,") + defer m.handle.Sync(true) - info.UID = msg.UID - if err := txTgt.Save(&info); err != nil { - return fmt.Errorf("CopyMessages %s, %d (tgt UID): info save: %w", tgtMbox.name, msg.UID, err) + seqset, err = m.handle.ResolveSeq(uid, seqset) + if err != nil { + if uid { + return nil } + return err + } - cache.UID = msg.UID - if err := txTgt.Save(&cache); err != nil { - return fmt.Errorf("CopyMessages %s, %d (tgt UID): cache save: %w", tgtMbox.name, msg.UID, err) + for _, entry := range entries { + if !seqset.Contains(entry.uid) { + continue } - flags.UID = msg.UID - if err := txTgt.Save(&flags); err != nil { - return fmt.Errorf("CopyMessages %s, %d (tgt UID): flags save: %w", tgtMbox.name, msg.UID, err) + src, err := entry.msg.Open() + if err != nil { + continue } - - if err := os.Link(srcName, tgtName); err != nil { - return fmt.Errorf("CopyMessages %s, %d (tgt UID): %w", tgtMbox.name, msg.UID, err) + copied, writer, err := destMbox.dir.Create(destMbox.maildirFlagsFromImap(entry.meta.flags)) + if err != nil { + src.Close() + return errors.New("I/O error, try again later") + } + if _, err := io.Copy(writer, src); err != nil { + _ = writer.Close() + src.Close() + return errors.New("I/O error, try again later") } - purgeList = append(purgeList, tgtName) + if err := writer.Close(); err != nil { + src.Close() + return errors.New("I/O error, try again later") + } + _ = src.Close() - return nil - }) - if err != nil { - for _, f := range purgeList { - if err := os.Remove(f); err != nil { - m.error("purgeList %s", err, tgtMbox.Name) + if !entry.meta.internalDate.IsZero() { + if err := copied.Chtimes(entry.meta.internalDate, entry.meta.internalDate); err != nil { + m.b.Log.Printf("CopyMessages: chtimes: %v", err) } } - if err == storm.ErrNotFound { - if uid { - return nil - } - return errors.New("No messages") + + m.b.statesLock.Lock() + uid := destMbox.state.meta.UIDNext + destMbox.state.meta.UIDNext++ + destMbox.state.meta.UIDByKey[copied.Key()] = uid + destMbox.state.meta.FilenameByUID[uid] = copied.Name() + destMbox.state.meta.DirtyUIDList = true + destMbox.state.meta.DirtyUIDList = true + meta := &messageMeta{ + uid: uid, + flags: append([]string{}, entry.meta.flags...), + internalDate: entry.meta.internalDate, } - m.b.Log.Println(err) - return errors.New("I/O error, try again later") + destMbox.state.uidNext = destMbox.state.meta.UIDNext + destMbox.state.messages[copied.Key()] = meta + storeRecent := m.b.Manager.NewMessage(destMbox.mailboxKey(), meta.uid) + meta.recent = storeRecent + m.b.statesLock.Unlock() } - var mboxInfo mboxData - if err := txTgt.One("Dummy", 1, &mboxInfo); err != nil { - m.error("CopyMesages: target info load", err) - return errors.New("I/O error, try again later") - } - mboxInfo.MsgsCount += uint32(len(purgeList)) - if err := txTgt.Save(&mboxInfo); err != nil { - m.error("CopyMesages: target info save", err) + if err := destMbox.writeMetadataState(destMbox.state.meta); err != nil { return errors.New("I/O error, try again later") } - if err := txTgt.Commit(); err != nil { - m.error("CopyMessages tgt: commit", err) - rollbackFiles() + return nil +} + +func (m *SelectedMailbox) MoveMessages(uid bool, seqset *imap.SeqSet, dest string) error { + destMbox, ok := m.user.getMailbox(dest) + if !ok { + if err := m.user.validateMboxName(dest); err != nil { + return err + } + mboxDir, err := m.user.storage.Dir(dest) + if err != nil { + return err + } + exists, err := mboxDir.Exists() + if err != nil { + return errors.New("I/O error, try again later") + } + if !exists { + return backend.ErrNoSuchMailbox + } + m.user.mailboxesLock.Lock() + destMbox = m.user.ensureMailbox(dest) + m.user.mailboxesLock.Unlock() + } + if err := destMbox.loadMetadataState(); err != nil { return errors.New("I/O error, try again later") } - if err := txSrc.Commit(); err != nil { - m.error("CopyMessages: commit src", err) - rollbackFiles() + + entries, err := m.listEntries() + if err != nil { return errors.New("I/O error, try again later") } - return nil -} + defer m.handle.Sync(true) -func (m *Mailbox) Expunge() error { - var msgs []message - if err := m.handle.AllByIndex("UID", &msgs); err != nil { - m.error("I/O error", err) - return errors.New("I/O error") + seqset, err = m.handle.ResolveSeq(uid, seqset) + if err != nil { + if uid { + return nil + } + return err } - errored := false - - for _, msg := range msgs { - if !msg.Deleted { + for _, entry := range entries { + if !seqset.Contains(entry.uid) { continue } - if err := m.dir.Remove(msg.key()); err != nil { - errored = true - m.error("I/O error", err) + src, err := entry.msg.Open() + if err != nil { continue } - if err := m.handle.DeleteStruct(&msg); err != nil { - errored = true - m.error("I/O error", err) - continue + copied, writer, err := destMbox.dir.Create(destMbox.maildirFlagsFromImap(entry.meta.flags)) + if err != nil { + src.Close() + return errors.New("I/O error, try again later") } + if _, err := io.Copy(writer, src); err != nil { + _ = writer.Close() + src.Close() + return errors.New("I/O error, try again later") + } + if err := writer.Close(); err != nil { + src.Close() + return errors.New("I/O error, try again later") + } + _ = src.Close() - if err := m.handle.Delete("messageInfo", msg.UID); err != nil { - errored = true - m.error("I/O error", err) - continue + if !entry.meta.internalDate.IsZero() { + if err := copied.Chtimes(entry.meta.internalDate, entry.meta.internalDate); err != nil { + m.b.Log.Printf("MoveMessages: chtimes: %v", err) + } } - if err := m.handle.Delete("messageCache", msg.UID); err != nil { - errored = true - m.error("I/O error", err) - continue + + m.b.statesLock.Lock() + uid := destMbox.state.meta.UIDNext + destMbox.state.meta.UIDNext++ + destMbox.state.meta.UIDByKey[copied.Key()] = uid + destMbox.state.meta.FilenameByUID[uid] = copied.Name() + destMbox.state.meta.DirtyUIDList = true + destMbox.state.meta.DirtyUIDList = true + meta := &messageMeta{ + uid: uid, + flags: append([]string{}, entry.meta.flags...), + internalDate: entry.meta.internalDate, } + destMbox.state.uidNext = destMbox.state.meta.UIDNext + destMbox.state.messages[copied.Key()] = meta + storeRecent := m.b.Manager.NewMessage(destMbox.mailboxKey(), meta.uid) + meta.recent = storeRecent + delete(m.state.messages, entry.msg.Key()) + m.b.statesLock.Unlock() + + if err := entry.msg.Remove(); err != nil { + m.b.Log.Printf("MoveMessages: remove: %v", err) + } + + m.handle.Removed(entry.uid) } - // TODO: Emit accumulated ExpungeUpdate. - if errored { - return errors.New("I/O error occured during expunge operation, not all messages are removed") + if err := destMbox.writeMetadataState(destMbox.state.meta); err != nil { + return errors.New("I/O error, try again later") } + return nil } -func (m *Mailbox) Close() error { - // XXX: This function is currently not called by go-imap. - // https://github.com/emersion/go-imap/pull/341 - // Mailbox handles __WILL__ leak. - - if m.handle == nil { - return nil +func (m *SelectedMailbox) Expunge() error { + entries, err := m.listEntries() + if err != nil { + return fmt.Errorf("I/O error: %w", err) } - m.b.dbsLock.Lock() - defer m.b.dbsLock.Unlock() - - key := m.username + "\x00" + m.name + for _, entry := range entries { + if !hasFlag(entry.meta.flags, imap.DeletedFlag) { + continue + } + if err := entry.msg.Remove(); err != nil { + m.b.Log.Printf("Expunge: %v", err) + continue + } - handle := m.b.dbs[key] - handle.uses-- + m.b.statesLock.Lock() + delete(m.state.messages, entry.msg.Key()) + m.b.statesLock.Unlock() - // Some sanity checks. - if handle.uses < 0 { - m.error("mailbox %s/%s: BUG: BoltDB reference counter went negative for", nil) + m.handle.Removed(entry.uid) } - if handle.db != m.handle { - m.error("mailbox %s/%s: BUG: multiple database handles created", nil) + + m.handle.Sync(true) + return nil +} + +func (m *Mailbox) messageEntity(msg maildir.Message) (*message.Entity, error) { + file, err := msg.Open() + if err != nil { + return nil, err } + defer file.Close() - if handle.uses <= 0 { - delete(m.b.dbs, key) - if err := m.handle.Close(); err != nil { - m.error("close failed: %v", err) - return err - } - return nil + data, err := io.ReadAll(file) + if err != nil { + return nil, err } - m.b.dbs[key] = handle - return nil + return message.Read(bytes.NewReader(data)) +} + +func containsFetchItem(items []imap.FetchItem, item imap.FetchItem) bool { + return slices.Contains(items, item) } diff --git a/maildir/errors.go b/maildir/errors.go new file mode 100644 index 0000000..c0b62ab --- /dev/null +++ b/maildir/errors.go @@ -0,0 +1,9 @@ +package maildir + +import "errors" + +var ErrNotExist = errors.New("maildir: not exist") + +func IsNotExist(err error) bool { + return errors.Is(err, ErrNotExist) +} diff --git a/maildir/fs/fs.go b/maildir/fs/fs.go new file mode 100644 index 0000000..f94b0d2 --- /dev/null +++ b/maildir/fs/fs.go @@ -0,0 +1,408 @@ +package fs + +import ( + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "github.com/emersion/go-maildir" + + imapmaildir "github.com/foxcpp/go-imap-maildir/maildir" +) + +const ( + hierarchySep = "." + inboxName = "INBOX" + maxNesting = 100 +) + +type Storage struct { + basePath string +} + +func (s *Storage) Dir(name string) (imapmaildir.Dir, error) { + path, err := s.pathForMailbox(name) + if err != nil { + return nil, err + } + return &Dir{ + dir: maildir.Dir(path), + basePath: s.basePath, + name: name, + }, nil +} + +func (s *Storage) ListDirs() ([]string, error) { + var paths []string + + err := filepath.WalkDir(s.basePath, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return nil + } + if !entry.IsDir() { + return nil + } + if path == s.basePath { + return nil + } + if !strings.HasPrefix(entry.Name(), ".") { + return filepath.SkipDir + } + paths = append(paths, path) + return nil + }) + if err != nil { + return nil, err + } + + mboxes := make([]string, 0, len(paths)) + for _, path := range paths { + name, err := s.mboxName(path) + if err != nil { + continue + } + if strings.EqualFold(name, inboxName) { + continue + } + mboxes = append(mboxes, name) + } + + return mboxes, nil +} + +func (s *Storage) Attributes() (imapmaildir.AttributesStore, error) { + return &attributesStore{basePath: s.basePath}, nil +} + +func (s *Storage) pathForMailbox(name string) (string, error) { + if strings.EqualFold(name, inboxName) { + return s.basePath, nil + } + parts := strings.Split(name, hierarchySep) + if len(parts) > maxNesting { + return "", errors.New("mailbox nesting limit exceeded") + } + currentPath := s.basePath + for i, part := range parts { + if i == 0 && strings.EqualFold(part, inboxName) { + continue + } + if part == "" { + if i != len(parts)-1 { + return "", errors.New("illegal mailbox name") + } + continue + } + if !validMboxPart(part) { + return "", errors.New("illegal mailbox name") + } + currentPath = filepath.Join(currentPath, "."+part) + } + return currentPath, nil +} + +func (s *Storage) mboxName(fsPath string) (string, error) { + fsPath = strings.TrimPrefix(fsPath, s.basePath+string(filepath.Separator)) + if fsPath == "" { + return inboxName, nil + } + parts := strings.Split(fsPath, string(filepath.Separator)) + if len(parts) > maxNesting { + return "", errors.New("mailbox nesting limit exceeded") + } + mboxParts := make([]string, 0, len(parts)) + for _, part := range parts { + if !strings.HasPrefix(part, ".") { + return "", errors.New("not a maildir++ path") + } + mboxParts = append(mboxParts, part[1:]) + } + return strings.Join(mboxParts, hierarchySep), nil +} + +type attributesStore struct { + basePath string +} + +func (s *attributesStore) Read() (map[string]string, error) { + return readAttributes(s.basePath) +} + +func (s *attributesStore) Write(attrs map[string]string) error { + return writeAttributes(s.basePath, attrs) +} + +type Dir struct { + dir maildir.Dir + basePath string + name string +} + +func (d *Dir) Name() string { + return d.name +} + +func (d *Dir) Path() string { + return string(d.dir) +} + +func (d *Dir) NewDelivery() (imapmaildir.Delivery, error) { + wrapped, err := maildir.NewDelivery(string(d.dir)) + if err != nil { + return nil, err + } + return &delivery{wrapped: wrapped}, nil +} + +func (d *Dir) Init() error { + if err := os.MkdirAll(string(d.dir), 0700); err != nil { + return err + } + return d.dir.Init() +} + +func (d *Dir) Exists() (bool, error) { + _, err := os.Stat(string(d.dir)) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func (d *Dir) Unseen() ([]imapmaildir.Message, error) { + msgs, err := d.dir.Unseen() + if err != nil { + if os.IsNotExist(err) { + return nil, imapmaildir.ErrNotExist + } + return nil, err + } + return wrapMessages(msgs), nil +} + +func (d *Dir) Messages() ([]imapmaildir.Message, error) { + msgs, err := d.dir.Messages() + if err != nil { + if os.IsNotExist(err) { + return nil, imapmaildir.ErrNotExist + } + return nil, err + } + return wrapMessages(msgs), nil +} + +func (d *Dir) Create(flags []imapmaildir.Flag) (imapmaildir.Message, io.WriteCloser, error) { + emFlags := make([]maildir.Flag, len(flags)) + for i, flag := range flags { + emFlags[i] = maildir.Flag(flag) + } + msg, writer, err := d.dir.Create(emFlags) + if err != nil { + return nil, nil, err + } + return &message{msg: msg}, writer, nil +} + +type delivery struct { + wrapped *maildir.Delivery +} + +func (d *delivery) Write(p []byte) (int, error) { + return d.wrapped.Write(p) +} + +func (d *delivery) Close() error { + return d.wrapped.Close() +} + +func (d *delivery) Abort() error { + return d.wrapped.Abort() +} + +func (d *Dir) Children() ([]imapmaildir.Dir, error) { + entries, err := os.ReadDir(string(d.dir)) + if err != nil { + if os.IsNotExist(err) { + return nil, imapmaildir.ErrNotExist + } + return nil, err + } + + children := make([]imapmaildir.Dir, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + name := entry.Name() + if name == "cur" || name == "new" || name == "tmp" { + continue + } + if !strings.HasPrefix(name, ".") { + continue + } + childPath := filepath.Join(string(d.dir), name) + childName := d.childName(name[1:]) + children = append(children, &Dir{dir: maildir.Dir(childPath), basePath: d.basePath, name: childName}) + } + + return children, nil +} + +func (d *Dir) Remove(childExists bool) error { + if childExists { + for _, name := range []string{"cur", "new", "tmp"} { + if err := os.RemoveAll(filepath.Join(string(d.dir), name)); err != nil && !os.IsNotExist(err) { + return err + } + } + return nil + } + return os.RemoveAll(string(d.dir)) +} + +func (d *Dir) Rename(name string) error { + fullName := d.renameTarget(name) + if fullName == "" { + return errors.New("illegal mailbox name") + } + path, err := (&Storage{basePath: d.basePath}).pathForMailbox(fullName) + if err != nil { + return err + } + return os.Rename(string(d.dir), path) +} + +func (d *Dir) LoadMetadata(state *imapmaildir.Metadata) error { + return newMetadataStore(string(d.dir)).Load(state) +} + +func (d *Dir) WriteMetadata(state *imapmaildir.Metadata) error { + return newMetadataStore(string(d.dir)).Write(state) +} + +func (d *Dir) childName(child string) string { + if strings.EqualFold(d.name, inboxName) { + return child + } + if d.name == "" { + return child + } + return d.name + hierarchySep + child +} + +func (d *Dir) renameTarget(name string) string { + if strings.EqualFold(d.name, inboxName) { + return name + } + if d.name == "" { + return name + } + parts := strings.Split(d.name, hierarchySep) + if len(parts) <= 1 { + return name + } + parent := strings.Join(parts[:len(parts)-1], hierarchySep) + if parent == "" { + return name + } + return parent + hierarchySep + name +} + +type message struct { + msg *maildir.Message +} + +func (m *message) Key() string { + return m.msg.Key() +} + +func (m *message) Name() string { + return m.msg.Filename() +} + +func (m *message) Flags() []imapmaildir.Flag { + flags := m.msg.Flags() + result := make([]imapmaildir.Flag, len(flags)) + for i, flag := range flags { + result[i] = imapmaildir.Flag(flag) + } + return result +} + +func (m *message) SetFlags(flags []imapmaildir.Flag) error { + emFlags := make([]maildir.Flag, len(flags)) + for i, flag := range flags { + emFlags[i] = maildir.Flag(flag) + } + return m.msg.SetFlags(emFlags) +} + +func (m *message) Open() (io.ReadCloser, error) { + return m.msg.Open() +} + +func (m *message) Stat() (imapmaildir.Info, error) { + info, err := os.Stat(m.msg.Filename()) + if err != nil { + if os.IsNotExist(err) { + return nil, imapmaildir.ErrNotExist + } + return nil, err + } + return info, nil +} + +func (m *message) Chtimes(atime, mtime time.Time) error { + return os.Chtimes(m.msg.Filename(), atime, mtime) +} + +func (m *message) Remove() error { + return m.msg.Remove() +} + +func (m *message) MoveTo(target imapmaildir.Dir) error { + emDir, ok := target.(*Dir) + if !ok { + return errors.New("maildir: unsupported target dir implementation") + } + return m.msg.MoveTo(emDir.dir) +} + +func (m *message) CopyTo(target imapmaildir.Dir) (imapmaildir.Message, error) { + emDir, ok := target.(*Dir) + if !ok { + return nil, errors.New("maildir: unsupported target dir implementation") + } + newMsg, err := m.msg.CopyTo(emDir.dir) + if err != nil { + return nil, err + } + return &message{msg: newMsg}, nil +} + +func wrapMessages(msgs []*maildir.Message) []imapmaildir.Message { + result := make([]imapmaildir.Message, len(msgs)) + for i, msg := range msgs { + result[i] = &message{msg: msg} + } + return result +} + +func validMboxPart(name string) bool { + if strings.ContainsAny(name, ":*?\"<>|") { + return false + } + for _, ch := range name { + if ch < ' ' { + return false + } + } + return !strings.Contains(name, "..") +} diff --git a/maildir/fs/fs_test.go b/maildir/fs/fs_test.go new file mode 100644 index 0000000..60d5638 --- /dev/null +++ b/maildir/fs/fs_test.go @@ -0,0 +1,106 @@ +package fs + +import ( + "bytes" + "os" + "strings" + "testing" + + "github.com/foxcpp/go-imap-maildir/maildir" +) + +func TestFSDelivery(t *testing.T) { + storage, cleanup := newTestStorage(t) + defer cleanup() + + dir := mustInitDir(t, storage, "INBOX") + delivery, err := dir.NewDelivery() + if err != nil { + t.Fatal(err) + } + if _, err := delivery.Write([]byte("hello")); err != nil { + _ = delivery.Abort() + t.Fatal(err) + } + if err := delivery.Close(); err != nil { + t.Fatal(err) + } + + msgs, err := dir.Unseen() + if err != nil { + t.Fatal(err) + } + if len(msgs) != 1 { + t.Fatalf("expected 1 message in new, got %d", len(msgs)) + } + if readMessage(t, msgs[0]) != "hello" { + t.Fatalf("expected delivered message content") + } +} + +func TestFSDeliveryAbort(t *testing.T) { + storage, cleanup := newTestStorage(t) + defer cleanup() + + dir := mustInitDir(t, storage, "INBOX") + delivery, err := dir.NewDelivery() + if err != nil { + t.Fatal(err) + } + if _, err := delivery.Write([]byte("discard")); err != nil { + _ = delivery.Abort() + t.Fatal(err) + } + if err := delivery.Abort(); err != nil { + t.Fatal(err) + } + + msgs, err := dir.Unseen() + if err != nil { + t.Fatal(err) + } + if len(msgs) != 0 { + t.Fatalf("expected 0 messages in new, got %d", len(msgs)) + } +} + +func newTestStorage(t *testing.T) (maildir.Storage, func()) { + root, err := os.MkdirTemp("", "go-imap-maildir-fs-") + if err != nil { + t.Fatal(err) + } + storage := Provider{}.Storage(root) + cleanup := func() { + _ = os.RemoveAll(root) + } + return storage, cleanup +} + +func mustInitDir(t *testing.T, storage maildir.Storage, name string) maildir.Dir { + dir := mustDir(t, storage, name) + if err := dir.Init(); err != nil { + t.Fatal(err) + } + return dir +} + +func mustDir(t *testing.T, storage maildir.Storage, name string) maildir.Dir { + dir, err := storage.Dir(name) + if err != nil { + t.Fatal(err) + } + return dir +} + +func readMessage(t *testing.T, msg maildir.Message) string { + r, err := msg.Open() + if err != nil { + t.Fatal(err) + } + defer r.Close() + buf := &bytes.Buffer{} + if _, err := buf.ReadFrom(r); err != nil { + t.Fatal(err) + } + return strings.TrimSpace(buf.String()) +} diff --git a/maildir/fs/metadata.go b/maildir/fs/metadata.go new file mode 100644 index 0000000..4e7d3fb --- /dev/null +++ b/maildir/fs/metadata.go @@ -0,0 +1,233 @@ +package fs + +import ( + "bytes" + "os" + "path/filepath" + "time" + + "github.com/foxcpp/go-imap-maildir/maildir" + "github.com/foxcpp/go-imap-maildir/maildir/metadata" +) + +const ( + legacyUIDListFileName = "dovecot-uidlist" + legacyUIDListLockName = "dovecot-uidlist.lock" + legacyKeywordsFileName = "dovecot-keywords" + legacyAttributesName = "dovecot-attributes" +) + +type metadataStore struct { + basePath string +} + +func newMetadataStore(basePath string) *metadataStore { + return &metadataStore{basePath: basePath} +} + +func (s *metadataStore) controlPath(name string) string { + return filepath.Join(s.basePath, name) +} + +func (s *metadataStore) resolveControlNames(state *maildir.Metadata) { + state.SetUIDListName(metadata.UIDListFileName) + state.SetKeywordsName(metadata.KeywordsFileName) +} + +func (s *metadataStore) Load(state *maildir.Metadata) error { + if state.Loaded { + return nil + } + state.Ensure() + s.resolveControlNames(state) + + if err := s.readUidlist(state); err != nil { + return err + } + if err := s.readKeywords(state); err != nil { + return err + } + + state.Loaded = true + return nil +} + +func (s *metadataStore) readUidlist(state *maildir.Metadata) error { + path := s.controlPath(metadata.UIDListFileName) + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + legacyPath := s.controlPath(legacyUIDListFileName) + legacyFile, legacyErr := os.Open(legacyPath) + if legacyErr != nil { + if os.IsNotExist(legacyErr) { + metadata.InitUIDList(state) + return nil + } + return legacyErr + } + defer legacyFile.Close() + return metadata.ParseUIDList(legacyFile, state) + } + return err + } + defer file.Close() + return metadata.ParseUIDList(file, state) +} + +func (s *metadataStore) readKeywords(state *maildir.Metadata) error { + path := s.controlPath(metadata.KeywordsFileName) + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + legacyPath := s.controlPath(legacyKeywordsFileName) + legacyFile, legacyErr := os.Open(legacyPath) + if legacyErr != nil { + if os.IsNotExist(legacyErr) { + return nil + } + return legacyErr + } + defer legacyFile.Close() + return metadata.ReadKeywords(legacyFile, state) + } + return err + } + defer file.Close() + return metadata.ReadKeywords(file, state) +} + +func (s *metadataStore) Write(state *maildir.Metadata) error { + if !state.DirtyUIDList && !state.DirtyKeywords { + return nil + } + return s.withUidlistLock(state, func() error { + if state.DirtyKeywords { + if err := s.writeKeywords(state); err != nil { + return err + } + state.DirtyKeywords = false + } + if state.DirtyUIDList { + if err := s.writeUidlist(state); err != nil { + return err + } + state.DirtyUIDList = false + } + return nil + }) +} + +func (s *metadataStore) writeUidlist(state *maildir.Metadata) error { + path := s.controlPath(state.UIDListName()) + tmp := path + ".tmp" + + buf := &bytes.Buffer{} + if err := metadata.WriteUIDList(buf, state); err != nil { + return err + } + if err := os.WriteFile(tmp, buf.Bytes(), 0600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +func (s *metadataStore) writeKeywords(state *maildir.Metadata) error { + path := s.controlPath(state.KeywordsName()) + tmp := path + ".tmp" + + buf := &bytes.Buffer{} + if err := metadata.WriteKeywords(buf, state); err != nil { + return err + } + if err := os.WriteFile(tmp, buf.Bytes(), 0600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +func (s *metadataStore) withUidlistLock(state *maildir.Metadata, fn func() error) error { + lockName := metadata.UIDListLockFileName + lockPath := s.controlPath(lockName) + deadline := time.Now().Add(5 * time.Second) + for { + file, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err == nil { + _ = file.Close() + defer os.Remove(lockPath) + return fn() + } + if !os.IsExist(err) || time.Now().After(deadline) { + return err + } + time.Sleep(50 * time.Millisecond) + } +} + +func readAttributes(basePath string) (map[string]string, error) { + path := attributesPath(basePath) + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return map[string]string{}, nil + } + return nil, err + } + defer file.Close() + return metadata.ReadAttributes(file) +} + +func writeAttributes(basePath string, attrs map[string]string) error { + path := attributesPath(basePath) + return withAttributesLock(basePath, func() error { + buf := &bytes.Buffer{} + if err := metadata.WriteAttributes(buf, attrs); err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, buf.Bytes(), 0600); err != nil { + return err + } + return os.Rename(tmp, path) + }) +} + +func attributesPath(basePath string) string { + path := filepath.Join(basePath, metadata.AttributesFileName) + if _, err := os.Stat(path); err == nil { + return path + } + legacyPath := filepath.Join(basePath, legacyAttributesName) + if _, err := os.Stat(legacyPath); err == nil { + return legacyPath + } + return path +} + +func withAttributesLock(basePath string, fn func() error) error { + lockPath := attributesPath(basePath) + ".lock" + deadline := time.Now().Add(5 * time.Second) + for { + file, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err == nil { + _ = file.Close() + defer os.Remove(lockPath) + return fn() + } + if !os.IsExist(err) || time.Now().After(deadline) { + return err + } + time.Sleep(50 * time.Millisecond) + } +} + +func (s *metadataStore) controlExists(name string) (bool, error) { + _, err := os.Stat(filepath.Join(s.basePath, name)) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} diff --git a/maildir/fs/provider.go b/maildir/fs/provider.go new file mode 100644 index 0000000..373f723 --- /dev/null +++ b/maildir/fs/provider.go @@ -0,0 +1,14 @@ +package fs + +import "github.com/foxcpp/go-imap-maildir/maildir" + +type Provider struct { + RootDir string +} + +func (c Provider) Storage(basePath string) maildir.Storage { + if c.RootDir == "" { + return &Storage{basePath: basePath} + } + return &Storage{basePath: c.RootDir + "/" + basePath} +} diff --git a/maildir/maildir.go b/maildir/maildir.go new file mode 100644 index 0000000..854af94 --- /dev/null +++ b/maildir/maildir.go @@ -0,0 +1,73 @@ +package maildir + +import ( + "io" + "time" +) + +// Flag is a message flag. +type Flag rune + +const ( + FlagPassed Flag = 'P' + FlagReplied Flag = 'R' + FlagSeen Flag = 'S' + FlagTrashed Flag = 'T' + FlagDraft Flag = 'D' + FlagFlagged Flag = 'F' +) + +// Message represents a message in a Maildir. +type Message interface { + Key() string + Name() string + Flags() []Flag + SetFlags(flags []Flag) error + Open() (io.ReadCloser, error) + Stat() (Info, error) + Chtimes(atime, mtime time.Time) error + Remove() error + MoveTo(target Dir) error + CopyTo(target Dir) (Message, error) +} + +type Info interface { + Size() int64 + ModTime() time.Time +} + +// Dir represents a Maildir directory. +type Dir interface { + Name() string + Init() error + Exists() (bool, error) + Unseen() ([]Message, error) + Messages() ([]Message, error) + Create(flags []Flag) (Message, io.WriteCloser, error) + NewDelivery() (Delivery, error) + Children() ([]Dir, error) + Remove(childExists bool) error + Rename(name string) error + LoadMetadata(state *Metadata) error + WriteMetadata(state *Metadata) error +} + +type Storage interface { + Dir(name string) (Dir, error) + ListDirs() ([]string, error) + Attributes() (AttributesStore, error) +} + +type Provider interface { + Storage(basePath string) Storage +} + +type AttributesStore interface { + Read() (map[string]string, error) + Write(attrs map[string]string) error +} + +type Delivery interface { + io.WriteCloser + Abort() error +} diff --git a/maildir/metadata.go b/maildir/metadata.go new file mode 100644 index 0000000..b1eb7be --- /dev/null +++ b/maildir/metadata.go @@ -0,0 +1,89 @@ +package maildir + +import ( + "crypto/rand" + "fmt" + "time" +) + +type Metadata struct { + Loaded bool + + UIDValidity uint32 + UIDNext uint32 + GUID string + + UIDByKey map[string]uint32 + FilenameByUID map[uint32]string + + KeywordByName map[string]rune + NameByKeyword map[rune]string + + DirtyUIDList bool + DirtyKeywords bool + + uidlistName string + keywordsName string +} + +func (m *Metadata) Ensure() { + if m.UIDByKey == nil { + m.UIDByKey = map[string]uint32{} + } + if m.FilenameByUID == nil { + m.FilenameByUID = map[uint32]string{} + } + if m.KeywordByName == nil { + m.KeywordByName = map[string]rune{} + } + if m.NameByKeyword == nil { + m.NameByKeyword = map[rune]string{} + } +} + +func (m *Metadata) EnsureKeyword(keyword string) rune { + if letter, ok := m.KeywordByName[keyword]; ok { + return letter + } + for i := range 26 { + letter := rune('a' + i) + if _, ok := m.NameByKeyword[letter]; !ok { + m.KeywordByName[keyword] = letter + m.NameByKeyword[letter] = keyword + m.DirtyKeywords = true + return letter + } + } + return 0 +} + +func (m *Metadata) EnsureGUID() { + if m.GUID == "" { + m.GUID = newMailboxGUID() + m.DirtyUIDList = true + } +} + +func (m *Metadata) UIDListName() string { + return m.uidlistName +} + +func (m *Metadata) SetUIDListName(name string) { + m.uidlistName = name +} + +func (m *Metadata) KeywordsName() string { + return m.keywordsName +} + +func (m *Metadata) SetKeywordsName(name string) { + m.keywordsName = name +} + +func newMailboxGUID() string { + buf := make([]byte, 16) + if _, err := rand.Read(buf); err != nil { + return fmt.Sprintf("%x", time.Now().UnixNano()) + } + return fmt.Sprintf("%x", buf) +} diff --git a/maildir/metadata/metadata.go b/maildir/metadata/metadata.go new file mode 100644 index 0000000..8a03ed4 --- /dev/null +++ b/maildir/metadata/metadata.go @@ -0,0 +1,279 @@ +package metadata + +import ( + "bufio" + "errors" + "fmt" + "io" + "slices" + "sort" + "strconv" + "strings" + "time" + + "github.com/foxcpp/go-imap-maildir/maildir" +) + +const ( + UIDListFileName = "uidlist" + UIDListLockFileName = "uidlist.lock" + KeywordsFileName = "keywords" + AttributesFileName = "attributes" +) + +func ParseUIDList(r io.Reader, state *maildir.Metadata) error { + reader := bufio.NewReader(r) + line, err := reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return err + } + line = strings.TrimSpace(line) + if line != "" { + parseUIDListHeader(state, line) + } + + for { + line, err = reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return err + } + line = strings.TrimSpace(line) + if line != "" { + parseUIDListEntry(state, line) + } + if errors.Is(err, io.EOF) { + break + } + } + + FinalizeUIDList(state) + return nil +} + +func InitUIDList(state *maildir.Metadata) { + state.UIDValidity = uint32(time.Now().UnixNano()) + if state.UIDValidity == 0 { + state.UIDValidity = 1 + } + state.UIDNext = 1 +} + +func FinalizeUIDList(state *maildir.Metadata) { + if state.UIDNext == 0 { + var max uint32 + for _, uid := range state.UIDByKey { + if uid > max { + max = uid + } + } + state.UIDNext = max + 1 + } + + if state.UIDValidity == 0 { + state.UIDValidity = uint32(time.Now().UnixNano()) + if state.UIDValidity == 0 { + state.UIDValidity = 1 + } + } + if state.GUID == "" { + state.EnsureGUID() + } +} + +func WriteUIDList(w io.Writer, state *maildir.Metadata) error { + guid := state.GUID + if guid == "" { + state.EnsureGUID() + guid = state.GUID + } + if _, err := fmt.Fprintf(w, "3 V%d N%d G%s\n", state.UIDValidity, state.UIDNext, guid); err != nil { + return err + } + + uids := make([]uint32, 0, len(state.FilenameByUID)) + for uid := range state.FilenameByUID { + uids = append(uids, uid) + } + slices.Sort(uids) + + for _, uid := range uids { + name := state.FilenameByUID[uid] + if name == "" { + continue + } + if _, err := fmt.Fprintf(w, "%d :%s\n", uid, name); err != nil { + return err + } + } + + return nil +} + +func ReadKeywords(r io.Reader, state *maildir.Metadata) error { + reader := bufio.NewReader(r) + for { + line, err := reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return err + } + line = strings.TrimSpace(line) + if line != "" { + fields := strings.Fields(line) + if len(fields) >= 2 { + idx, err := strconv.Atoi(fields[0]) + if err == nil && idx >= 0 && idx < 26 { + keyword := strings.Join(fields[1:], " ") + letter := rune('a' + idx) + state.KeywordByName[keyword] = letter + state.NameByKeyword[letter] = keyword + } + } + } + if errors.Is(err, io.EOF) { + break + } + } + + return nil +} + +func WriteKeywords(w io.Writer, state *maildir.Metadata) error { + letters := make([]rune, 0, len(state.NameByKeyword)) + for letter := range state.NameByKeyword { + letters = append(letters, letter) + } + slices.Sort(letters) + + for _, letter := range letters { + idx := int(letter - 'a') + if idx < 0 || idx >= 26 { + continue + } + name := state.NameByKeyword[letter] + if _, err := fmt.Fprintf(w, "%d %s\n", idx, name); err != nil { + return err + } + } + + return nil +} + +func ReadAttributes(r io.Reader) (map[string]string, error) { + attrs := map[string]string{} + reader := bufio.NewReader(r) + for { + keyLine, err := reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + keyLine = strings.TrimRight(keyLine, "\r\n") + if keyLine == "" && errors.Is(err, io.EOF) { + break + } + valueLine, err2 := reader.ReadString('\n') + if err2 != nil && !errors.Is(err2, io.EOF) { + return nil, err2 + } + valueLine = strings.TrimRight(valueLine, "\r\n") + key := tabUnescape(keyLine) + value := tabUnescape(valueLine) + if key != "" { + attrs[key] = value + } + if errors.Is(err, io.EOF) || errors.Is(err2, io.EOF) { + break + } + } + return attrs, nil +} + +func WriteAttributes(w io.Writer, attrs map[string]string) error { + keys := make([]string, 0, len(attrs)) + for key := range attrs { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + value := attrs[key] + if _, err := fmt.Fprintln(w, tabEscape(key)); err != nil { + return err + } + if _, err := fmt.Fprintln(w, tabEscape(value)); err != nil { + return err + } + } + return nil +} + +func ParseMaildirKey(filename string) string { + key, _, _ := strings.Cut(filename, ":") + return key +} + +func BaseName(value string) string { + if idx := strings.LastIndex(value, "/"); idx >= 0 { + return value[idx+1:] + } + return value +} + +func parseUIDListHeader(state *maildir.Metadata, line string) { + fields := strings.Fields(line) + for _, field := range fields { + switch { + case strings.HasPrefix(field, "V"): + if val, err := strconv.ParseUint(field[1:], 10, 32); err == nil { + state.UIDValidity = uint32(val) + } + case strings.HasPrefix(field, "N"): + if val, err := strconv.ParseUint(field[1:], 10, 32); err == nil { + state.UIDNext = uint32(val) + } + case strings.HasPrefix(field, "G"): + state.GUID = field[1:] + } + } +} + +func parseUIDListEntry(state *maildir.Metadata, line string) { + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + return + } + left := strings.TrimSpace(parts[0]) + right := strings.TrimSpace(parts[1]) + if left == "" || right == "" { + return + } + fields := strings.Fields(left) + if len(fields) == 0 { + return + } + uid, err := strconv.ParseUint(fields[0], 10, 32) + if err != nil || uid == 0 { + return + } + filename := BaseName(right) + key := ParseMaildirKey(filename) + if key == "" { + return + } + state.UIDByKey[key] = uint32(uid) + state.FilenameByUID[uint32(uid)] = filename +} + +func tabEscape(value string) string { + value = strings.ReplaceAll(value, "\\", "\\\\") + value = strings.ReplaceAll(value, "\t", "\\t") + value = strings.ReplaceAll(value, "\n", "\\n") + value = strings.ReplaceAll(value, "\r", "\\r") + return value +} + +func tabUnescape(value string) string { + value = strings.ReplaceAll(value, "\\t", "\t") + value = strings.ReplaceAll(value, "\\n", "\n") + value = strings.ReplaceAll(value, "\\r", "\r") + value = strings.ReplaceAll(value, "\\\\", "\\") + return value +} diff --git a/maildir/migrate.go b/maildir/migrate.go new file mode 100644 index 0000000..ebb0e8a --- /dev/null +++ b/maildir/migrate.go @@ -0,0 +1,424 @@ +package maildir + +import ( + "errors" + "fmt" + "io" + "runtime" + "sync" + "time" +) + +func MigrateStorage(src, dst Storage) error { + if src == nil || dst == nil { + return errors.New("maildir: missing storage") + } + + if err := migrateAttributes(src, dst); err != nil { + return err + } + + mailboxes, err := listMailboxes(src) + if err != nil { + return err + } + if err := migrateMailboxes(src, dst, mailboxes); err != nil { + return err + } + + return nil +} + +func migrateMailboxes(src, dst Storage, mailboxes []string) error { + if len(mailboxes) == 0 { + return nil + } + workers := min(max(runtime.GOMAXPROCS(0), 1), len(mailboxes)) + + workCh := make(chan string) + var wg sync.WaitGroup + var firstErr error + var errOnce sync.Once + done := make(chan struct{}) + setErr := func(err error) { + errOnce.Do(func() { + firstErr = err + close(done) + }) + } + + worker := func() { + defer wg.Done() + for { + select { + case <-done: + return + case name, ok := <-workCh: + if !ok { + return + } + if err := migrateMailbox(src, dst, name); err != nil { + setErr(err) + } + } + } + } + + for range workers { + wg.Add(1) + go worker() + } +mailboxLoop: + for _, name := range mailboxes { + select { + case <-done: + break mailboxLoop + case workCh <- name: + } + } + close(workCh) + wg.Wait() + return firstErr +} + +func listMailboxes(src Storage) ([]string, error) { + names, err := src.ListDirs() + if err != nil { + return nil, fmt.Errorf("maildir: list mailboxes: %w", err) + } + + var result []string + seen := map[string]struct{}{} + addName := func(name string) { + if name == "" { + return + } + if _, ok := seen[name]; ok { + return + } + seen[name] = struct{}{} + result = append(result, name) + } + + if inbox, err := src.Dir("INBOX"); err == nil { + exists, err := inbox.Exists() + if err != nil { + return nil, fmt.Errorf("maildir: check inbox: %w", err) + } + if exists { + addName("INBOX") + } + } + for _, name := range names { + addName(name) + } + + queue := append([]string{}, result...) + for len(queue) > 0 { + name := queue[0] + queue = queue[1:] + dir, err := src.Dir(name) + if err != nil { + return nil, fmt.Errorf("maildir: open mailbox %s: %w", name, err) + } + children, err := dir.Children() + if err != nil { + if IsNotExist(err) { + continue + } + return nil, fmt.Errorf("maildir: list child mailboxes %s: %w", name, err) + } + for _, child := range children { + if child == nil { + continue + } + childName := child.Name() + if _, ok := seen[childName]; ok { + continue + } + addName(childName) + queue = append(queue, childName) + } + } + + return result, nil +} + +func migrateAttributes(src, dst Storage) error { + srcStore, err := src.Attributes() + if err != nil { + return fmt.Errorf("maildir: read attributes: %w", err) + } + attrs, err := srcStore.Read() + if err != nil { + return fmt.Errorf("maildir: read attributes: %w", err) + } + if len(attrs) == 0 { + return nil + } + dstStore, err := dst.Attributes() + if err != nil { + return fmt.Errorf("maildir: write attributes: %w", err) + } + if err := dstStore.Write(attrs); err != nil { + return fmt.Errorf("maildir: write attributes: %w", err) + } + return nil +} + +func migrateMailbox(src, dst Storage, name string) error { + srcDir, err := src.Dir(name) + if err != nil { + return fmt.Errorf("maildir: open source mailbox %s: %w", name, err) + } + exists, err := srcDir.Exists() + if err != nil { + return fmt.Errorf("maildir: check source mailbox %s: %w", name, err) + } + if !exists { + return nil + } + + dstDir, err := dst.Dir(name) + if err != nil { + return fmt.Errorf("maildir: open destination mailbox %s: %w", name, err) + } + if err := dstDir.Init(); err != nil { + return fmt.Errorf("maildir: init destination mailbox %s: %w", name, err) + } + if err := ensureEmptyMailbox(dstDir, name); err != nil { + return err + } + + srcMeta := &Metadata{} + if err := srcDir.LoadMetadata(srcMeta); err != nil { + return fmt.Errorf("maildir: read metadata %s: %w", name, err) + } + + dstMeta := &Metadata{} + dstMeta.Ensure() + copyKeywords(dstMeta, srcMeta) + dstMeta.UIDValidity = srcMeta.UIDValidity + dstMeta.UIDNext = srcMeta.UIDNext + dstMeta.GUID = srcMeta.GUID + dstMeta.SetUIDListName(srcMeta.UIDListName()) + dstMeta.SetKeywordsName(srcMeta.KeywordsName()) + + maxUID := uint32(0) + for _, uid := range srcMeta.UIDByKey { + if uid > maxUID { + maxUID = uid + } + } + if dstMeta.UIDNext == 0 { + dstMeta.UIDNext = maxUID + 1 + if dstMeta.UIDNext == 0 { + dstMeta.UIDNext = 1 + } + } + + msgs, err := srcDir.Messages() + if err != nil { + if !IsNotExist(err) { + return fmt.Errorf("maildir: list messages %s: %w", name, err) + } + msgs = nil + } + unseen, err := srcDir.Unseen() + if err != nil { + if !IsNotExist(err) { + return fmt.Errorf("maildir: list unseen %s: %w", name, err) + } + unseen = nil + } + + allMessages := make([]Message, 0, len(msgs)+len(unseen)) + seen := map[string]struct{}{} + for _, msg := range msgs { + if msg == nil { + continue + } + key := msg.Key() + if key == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + allMessages = append(allMessages, msg) + } + for _, msg := range unseen { + if msg == nil { + continue + } + key := msg.Key() + if key == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + allMessages = append(allMessages, msg) + } + + if err := migrateMessages(dstDir, allMessages, srcMeta, dstMeta, &maxUID); err != nil { + return fmt.Errorf("maildir: migrate message %s: %w", name, err) + } + + if dstMeta.UIDNext <= maxUID { + dstMeta.UIDNext = maxUID + 1 + } + if dstMeta.UIDValidity == 0 { + dstMeta.UIDValidity = uint32(time.Now().UnixNano()) + if dstMeta.UIDValidity == 0 { + dstMeta.UIDValidity = 1 + } + } + if dstMeta.GUID == "" { + dstMeta.EnsureGUID() + } + if len(dstMeta.KeywordByName) > 0 { + dstMeta.DirtyKeywords = true + } + dstMeta.DirtyUIDList = true + dstMeta.Loaded = true + + if err := dstDir.WriteMetadata(dstMeta); err != nil { + return fmt.Errorf("maildir: write metadata %s: %w", name, err) + } + + return nil +} + +func ensureEmptyMailbox(dir Dir, name string) error { + msgs, err := dir.Messages() + if err != nil && !IsNotExist(err) { + return fmt.Errorf("maildir: check destination mailbox %s: %w", name, err) + } + if len(msgs) > 0 { + return fmt.Errorf("maildir: destination mailbox %s is not empty", name) + } + return nil +} + +func migrateMessages(dst Dir, messages []Message, srcMeta, dstMeta *Metadata, maxUID *uint32) error { + if len(messages) == 0 { + return nil + } + workers := min(max(runtime.GOMAXPROCS(0), 1), len(messages)) + + workCh := make(chan Message) + var wg sync.WaitGroup + var firstErr error + var errOnce sync.Once + done := make(chan struct{}) + setErr := func(err error) { + errOnce.Do(func() { + firstErr = err + close(done) + }) + } + var metaLock sync.Mutex + + worker := func() { + defer wg.Done() + for { + select { + case <-done: + return + case msg, ok := <-workCh: + if !ok { + return + } + dstMsg, err := copyMessage(dst, msg) + if err != nil { + setErr(err) + continue + } + metaLock.Lock() + updateMessageMetadata(dstMsg, msg, srcMeta, dstMeta, maxUID) + metaLock.Unlock() + } + } + } + + for range workers { + wg.Add(1) + go worker() + } +messageLoop: + for _, msg := range messages { + select { + case <-done: + break messageLoop + case workCh <- msg: + } + } + close(workCh) + wg.Wait() + return firstErr +} + +func copyMessage(dst Dir, srcMsg Message) (Message, error) { + flags := srcMsg.Flags() + dstMsg, writer, err := dst.Create(flags) + if err != nil { + return nil, err + } + srcReader, err := srcMsg.Open() + if err != nil { + _ = writer.Close() + return nil, err + } + _, copyErr := io.Copy(writer, srcReader) + closeErr := srcReader.Close() + writeErr := writer.Close() + if copyErr != nil { + return nil, copyErr + } + if closeErr != nil { + return nil, closeErr + } + if writeErr != nil { + return nil, writeErr + } + if info, err := srcMsg.Stat(); err == nil { + _ = dstMsg.Chtimes(info.ModTime(), info.ModTime()) + } + return dstMsg, nil +} + +func updateMessageMetadata(dstMsg, srcMsg Message, srcMeta, dstMeta *Metadata, maxUID *uint32) { + srcUID := dstMeta.UIDNext + if uid, ok := srcMeta.UIDByKey[srcMsg.Key()]; ok && uid != 0 { + srcUID = uid + } else if uid, ok := srcMeta.UIDByKey[srcMsg.Name()]; ok && uid != 0 { + srcUID = uid + } + if uid := srcUID; uid == dstMeta.UIDNext { + if uid <= *maxUID { + uid = *maxUID + 1 + } + *maxUID = uid + srcUID = uid + } + if srcUID > *maxUID { + *maxUID = srcUID + } + dstMeta.UIDByKey[dstMsg.Key()] = srcUID + dstMeta.FilenameByUID[srcUID] = dstMsg.Name() +} + +func copyKeywords(dst, src *Metadata) { + if src == nil || dst == nil { + return + } + for key, value := range src.KeywordByName { + dst.KeywordByName[key] = value + } + for key, value := range src.NameByKeyword { + dst.NameByKeyword[key] = value + } +} diff --git a/maildir/s3/metadata.go b/maildir/s3/metadata.go new file mode 100644 index 0000000..5bf1911 --- /dev/null +++ b/maildir/s3/metadata.go @@ -0,0 +1,210 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "io" + "time" + + "github.com/minio/minio-go/v7" + + "github.com/foxcpp/go-imap-maildir/maildir" + "github.com/foxcpp/go-imap-maildir/maildir/metadata" +) + +type metadataStore struct { + storage *Storage + prefix string +} + +func newMetadataStore(storage *Storage, prefix string) *metadataStore { + return &metadataStore{storage: storage, prefix: prefix} +} + +func (s *metadataStore) controlKey(name string) string { + return joinKey(s.prefix, name) +} + +func (s *metadataStore) resolveControlNames(state *maildir.Metadata) { + state.SetUIDListName(metadata.UIDListFileName) + state.SetKeywordsName(metadata.KeywordsFileName) +} + +func (s *metadataStore) objectExists(key string) (bool, error) { + _, err := s.storage.client.StatObject(context.Background(), s.storage.bucket, key, minio.StatObjectOptions{}) + if err != nil { + if isNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (s *metadataStore) Load(state *maildir.Metadata) error { + if state.Loaded { + return nil + } + state.Ensure() + s.resolveControlNames(state) + + if err := s.readUidlist(state); err != nil { + return err + } + if err := s.readKeywords(state); err != nil { + return err + } + + state.Loaded = true + return nil +} + +func (s *metadataStore) readUidlist(state *maildir.Metadata) error { + key := s.controlKey(state.UIDListName()) + if cached, ok := s.storage.getCachedObject(key); ok { + return metadata.ParseUIDList(bytes.NewReader(cached), state) + } + obj, err := s.storage.client.GetObject(context.Background(), s.storage.bucket, key, minio.GetObjectOptions{}) + if err != nil { + if isNotFound(err) { + s.storage.deleteCachedObject(key) + metadata.InitUIDList(state) + return nil + } + return err + } + defer obj.Close() + if _, err := obj.Stat(); err != nil { + if isNotFound(err) { + s.storage.deleteCachedObject(key) + metadata.InitUIDList(state) + return nil + } + return err + } + data, err := io.ReadAll(obj) + if err != nil { + if isNotFound(err) { + s.storage.deleteCachedObject(key) + metadata.InitUIDList(state) + return nil + } + return err + } + s.storage.setCachedObject(key, data) + if err := metadata.ParseUIDList(bytes.NewReader(data), state); err != nil { + return err + } + return nil +} + +func (s *metadataStore) readKeywords(state *maildir.Metadata) error { + key := s.controlKey(state.KeywordsName()) + if cached, ok := s.storage.getCachedObject(key); ok { + return metadata.ReadKeywords(bytes.NewReader(cached), state) + } + obj, err := s.storage.client.GetObject(context.Background(), s.storage.bucket, key, minio.GetObjectOptions{}) + if err != nil { + if isNotFound(err) { + s.storage.deleteCachedObject(key) + return nil + } + return err + } + defer obj.Close() + if _, err := obj.Stat(); err != nil { + if isNotFound(err) { + s.storage.deleteCachedObject(key) + return nil + } + return err + } + data, err := io.ReadAll(obj) + if err != nil { + if isNotFound(err) { + s.storage.deleteCachedObject(key) + return nil + } + return err + } + s.storage.setCachedObject(key, data) + if err := metadata.ReadKeywords(bytes.NewReader(data), state); err != nil { + return err + } + return nil +} + +func (s *metadataStore) Write(state *maildir.Metadata) error { + if !state.DirtyUIDList && !state.DirtyKeywords { + return nil + } + return s.withUidlistLock(state, func() error { + if state.DirtyKeywords { + if err := s.writeKeywords(state); err != nil { + return err + } + state.DirtyKeywords = false + } + if state.DirtyUIDList { + if err := s.writeUidlist(state); err != nil { + return err + } + state.DirtyUIDList = false + } + return nil + }) +} + +func (s *metadataStore) writeUidlist(state *maildir.Metadata) error { + key := s.controlKey(state.UIDListName()) + buf := &bytes.Buffer{} + if err := metadata.WriteUIDList(buf, state); err != nil { + return err + } + _, err := s.storage.client.PutObject(context.Background(), s.storage.bucket, key, buf, int64(buf.Len()), minio.PutObjectOptions{}) + if err == nil { + s.storage.setCachedObject(key, buf.Bytes()) + } + return err +} + +func (s *metadataStore) writeKeywords(state *maildir.Metadata) error { + key := s.controlKey(state.KeywordsName()) + buf := &bytes.Buffer{} + if err := metadata.WriteKeywords(buf, state); err != nil { + return err + } + _, err := s.storage.client.PutObject(context.Background(), s.storage.bucket, key, buf, int64(buf.Len()), minio.PutObjectOptions{}) + if err == nil { + s.storage.setCachedObject(key, buf.Bytes()) + } + return err +} + +func (s *metadataStore) withUidlistLock(state *maildir.Metadata, fn func() error) error { + lockName := metadata.UIDListLockFileName + lockKey := s.controlKey(lockName) + deadline := time.Now().Add(5 * time.Second) + for { + exists, err := s.objectExists(lockKey) + if err != nil { + return err + } + if !exists { + _, err := s.storage.client.PutObject(context.Background(), s.storage.bucket, lockKey, bytes.NewReader(nil), 0, minio.PutObjectOptions{}) + if err == nil { + defer s.storage.client.RemoveObject(context.Background(), s.storage.bucket, lockKey, minio.RemoveObjectOptions{}) + return fn() + } + } + if time.Now().After(deadline) { + return errors.New("maildir: uidlist lock timeout") + } + time.Sleep(50 * time.Millisecond) + } +} + +func pathBase(value string) string { + return metadata.BaseName(value) +} diff --git a/maildir/s3/provider.go b/maildir/s3/provider.go new file mode 100644 index 0000000..4c40753 --- /dev/null +++ b/maildir/s3/provider.go @@ -0,0 +1,52 @@ +package s3 + +import ( + "github.com/dgraph-io/ristretto/v2" + "github.com/minio/minio-go/v7" + + "github.com/foxcpp/go-imap-maildir/maildir" +) + +func NewProvider(client *minio.Client, bucket, rootPrefix string) maildir.Provider { + cache := newCache(defaultCacheBytes) + return &Provider{ + Client: client, + Bucket: bucket, + RootPrefix: rootPrefix, + Cache: cache, + MaxItemSize: defaultCacheItemSize, + } +} + +func NewProviderWithCache(client *minio.Client, bucket, prefix string, cache *ristretto.Cache[string, []byte], maxItemSize int64) maildir.Provider { + if maxItemSize < 0 { + maxItemSize = 0 + } + return &Provider{ + Client: client, + Bucket: bucket, + RootPrefix: prefix, + Cache: cache, + MaxItemSize: maxItemSize, + } +} + +type Provider struct { + Client *minio.Client + Bucket string + RootPrefix string + + Cache *ristretto.Cache[string, []byte] + MaxItemSize int64 +} + +func (c *Provider) Storage(basePath string) maildir.Storage { + return &Storage{ + basePath: basePath, + client: c.Client, + bucket: c.Bucket, + rootPrefix: c.RootPrefix, + cache: c.Cache, + maxCacheItemSize: c.MaxItemSize, + } +} diff --git a/maildir/s3/s3.go b/maildir/s3/s3.go new file mode 100644 index 0000000..310e7bf --- /dev/null +++ b/maildir/s3/s3.go @@ -0,0 +1,1104 @@ +package s3 + +import ( + "bytes" + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "os" + "path" + "slices" + "sort" + "strings" + "time" + + "github.com/dgraph-io/ristretto/v2" + "github.com/minio/minio-go/v7" + + "github.com/foxcpp/go-imap-maildir/maildir" + "github.com/foxcpp/go-imap-maildir/maildir/metadata" +) + +const ( + hierarchySep = "." + inboxName = "INBOX" + maxNesting = 100 + registryName = "mailboxes" + mboxPrefixName = "mbox" + blobPrefixName = "blobs" + markerName = "mailbox" + + defaultCacheBytes int64 = 64 << 20 + defaultCacheItemSize int64 = 4 << 20 +) + +type Storage struct { + client *minio.Client + bucket string + rootPrefix string + basePath string + + cache *ristretto.Cache[string, []byte] + maxCacheItemSize int64 +} + +func (s *Storage) Dir(name string) (maildir.Dir, error) { + return &Dir{ + storage: s, + name: name, + }, nil +} + +func (s *Storage) ListDirs() ([]string, error) { + registryPrefix := s.registryPrefix() + keys, err := s.listKeys(registryPrefix) + if err != nil { + return nil, err + } + + seen := map[string]struct{}{} + for _, key := range keys { + rel := strings.TrimPrefix(key, registryPrefix) + rel = strings.TrimPrefix(rel, "/") + if rel == "" { + continue + } + seen[rel] = struct{}{} + } + + names := make([]string, 0, len(seen)) + for name := range seen { + if strings.EqualFold(name, inboxName) { + continue + } + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +func (s *Storage) listRegistryNames() ([]string, error) { + registryPrefix := s.registryPrefix() + keys, err := s.listKeys(registryPrefix) + if err != nil { + return nil, err + } + names := make([]string, 0, len(keys)) + for _, key := range keys { + rel := strings.TrimPrefix(key, registryPrefix) + rel = strings.TrimPrefix(rel, "/") + if rel == "" { + continue + } + names = append(names, rel) + } + return names, nil +} + +func (s *Storage) Attributes() (maildir.AttributesStore, error) { + return &attributesStore{storage: s}, nil +} + +func (s *Storage) registryPrefix() string { + return joinKey(s.rootPrefix, s.basePath, registryName) +} + +func (s *Storage) basePrefix() string { + return joinKey(s.rootPrefix, s.basePath) +} + +func (s *Storage) mboxRootPrefix() string { + return joinKey(s.rootPrefix, s.basePath, mboxPrefixName) +} + +func (s *Storage) blobRootPrefix() string { + return joinKey(s.rootPrefix, s.basePath, blobPrefixName) +} + +func (s *Storage) registryKey(name string) string { + return joinKey(s.registryPrefix(), normalizeMailboxName(name)) +} + +func (s *Storage) readRegistry(name string) (string, error) { + key := s.registryKey(name) + obj, err := s.client.GetObject(context.Background(), s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + if isNotFound(err) { + s.deleteCachedObject(key) + return "", maildir.ErrNotExist + } + return "", err + } + defer obj.Close() + if _, err := obj.Stat(); err != nil { + if isNotFound(err) { + s.deleteCachedObject(key) + return "", maildir.ErrNotExist + } + return "", err + } + data, err := io.ReadAll(obj) + if err != nil { + if isNotFound(err) { + s.deleteCachedObject(key) + return "", maildir.ErrNotExist + } + return "", err + } + s.setCachedObject(key, data) + guid := strings.TrimSpace(string(data)) + if guid == "" { + return "", errors.New("maildir: empty mailbox registry") + } + return guid, nil +} + +func (s *Storage) writeRegistry(name, guid string) error { + key := s.registryKey(name) + buf := bytes.NewBufferString(guid) + _, err := s.client.PutObject(context.Background(), s.bucket, key, buf, int64(buf.Len()), minio.PutObjectOptions{}) + if err == nil { + s.setCachedObject(key, []byte(guid)) + s.invalidateList(s.registryPrefix()) + } + return err +} + +func (s *Storage) deleteRegistry(name string) error { + key := s.registryKey(name) + err := s.client.RemoveObject(context.Background(), s.bucket, key, minio.RemoveObjectOptions{}) + if err == nil { + s.deleteCachedObject(key) + s.invalidateList(s.registryPrefix()) + } + return err +} + +func (s *Storage) listKeys(prefix string) ([]string, error) { + if keys, ok := s.getCachedList(prefix); ok { + return keys, nil + } + var keys []string + for obj := range s.client.ListObjects(context.Background(), s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if obj.Err != nil { + return nil, obj.Err + } + keys = append(keys, obj.Key) + } + s.setCachedList(prefix, keys) + return keys, nil +} + +func (s *Storage) removeKeys(keys []string) error { + if len(keys) == 0 { + return nil + } + objectsCh := make(chan minio.ObjectInfo) + go func() { + defer close(objectsCh) + for _, key := range keys { + if key == "" { + continue + } + objectsCh <- minio.ObjectInfo{Key: key} + } + }() + var firstErr error + for err := range s.client.RemoveObjects(context.Background(), s.bucket, objectsCh, minio.RemoveObjectsOptions{}) { + if err.Err == nil { + continue + } + if firstErr == nil { + firstErr = err.Err + } + } + return firstErr +} + +func (s *Storage) objectExists(key string) (bool, error) { + _, err := s.client.StatObject(context.Background(), s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + if isNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (s *Storage) getCachedObject(key string) ([]byte, bool) { + return s.getCachedBytes("obj:" + key) +} + +func (s *Storage) setCachedObject(key string, data []byte) { + s.setCachedBytes("obj:"+key, data) +} + +func (s *Storage) deleteCachedObject(key string) { + s.deleteCachedKey("obj:" + key) +} + +func (s *Storage) getCachedList(prefix string) ([]string, bool) { + data, ok := s.getCachedBytes("list:" + prefix) + if !ok { + return nil, false + } + return decodeListCache(data), true +} + +func (s *Storage) setCachedList(prefix string, keys []string) { + s.setCachedBytes("list:"+prefix, encodeListCache(keys)) +} + +func (s *Storage) invalidateList(prefix string) { + s.deleteCachedKey("list:" + prefix) +} + +func (s *Storage) invalidateListForKey(key string) { + if key == "" { + return + } + if idx := strings.Index(key, "/cur/"); idx > 0 { + base := key[:idx] + s.invalidateList(base) + s.invalidateList(base + "/cur") + return + } + if idx := strings.Index(key, "/new/"); idx > 0 { + base := key[:idx] + s.invalidateList(base) + s.invalidateList(base + "/new") + return + } +} + +func (s *Storage) getCachedBytes(key string) ([]byte, bool) { + if s.cache == nil { + return nil, false + } + data, ok := s.cache.Get(key) + if !ok { + return nil, false + } + return bytes.Clone(data), true +} + +func (s *Storage) setCachedBytes(key string, data []byte) { + if s.cache == nil || data == nil { + return + } + copyData := bytes.Clone(data) + _ = s.cache.Set(key, copyData, int64(len(copyData))) +} + +func (s *Storage) deleteCachedKey(key string) { + if s.cache == nil { + return + } + s.cache.Del(key) +} + +func encodeListCache(keys []string) []byte { + if len(keys) == 0 { + return nil + } + return []byte(strings.Join(keys, "\x1f")) +} + +func decodeListCache(data []byte) []string { + if len(data) == 0 { + return nil + } + return strings.Split(string(data), "\x1f") +} + +func newCache(size int64) *ristretto.Cache[string, []byte] { + if size <= 0 { + return nil + } + cache, err := ristretto.NewCache(&ristretto.Config[string, []byte]{ + NumCounters: size / 64, + MaxCost: size, + BufferItems: 64, + }) + if err != nil { + return nil + } + return cache +} + +func (s *Storage) prefixExists(prefix string) (bool, error) { + for obj := range s.client.ListObjects(context.Background(), s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if obj.Err != nil { + return false, obj.Err + } + return true, nil + } + return false, nil +} + +type attributesStore struct { + storage *Storage +} + +func (a *attributesStore) Read() (map[string]string, error) { + key := joinKey(a.storage.basePrefix(), metadata.AttributesFileName) + if cached, ok := a.storage.getCachedObject(key); ok { + return metadata.ReadAttributes(bytes.NewReader(cached)) + } + obj, err := a.storage.client.GetObject(context.Background(), a.storage.bucket, key, minio.GetObjectOptions{}) + if err != nil { + if isNotFound(err) { + a.storage.deleteCachedObject(key) + return map[string]string{}, nil + } + return nil, err + } + defer obj.Close() + if _, err := obj.Stat(); err != nil { + if isNotFound(err) { + a.storage.deleteCachedObject(key) + return map[string]string{}, nil + } + return nil, err + } + data, err := io.ReadAll(obj) + if err != nil { + if isNotFound(err) { + a.storage.deleteCachedObject(key) + return map[string]string{}, nil + } + return nil, err + } + a.storage.setCachedObject(key, data) + return metadata.ReadAttributes(bytes.NewReader(data)) +} + +func (a *attributesStore) Write(attrs map[string]string) error { + key := joinKey(a.storage.basePrefix(), metadata.AttributesFileName) + buf := &bytes.Buffer{} + if err := metadata.WriteAttributes(buf, attrs); err != nil { + return err + } + _, err := a.storage.client.PutObject(context.Background(), a.storage.bucket, key, buf, int64(buf.Len()), minio.PutObjectOptions{}) + if err == nil { + a.storage.setCachedObject(key, buf.Bytes()) + } + return err +} + +type Dir struct { + storage *Storage + name string + guid string + prefix string +} + +func (d *Dir) Name() string { + return d.name +} + +func (d *Dir) ensurePrefix(create bool) error { + if d.prefix != "" { + return nil + } + guid, err := d.storage.readRegistry(d.name) + if err != nil { + if maildir.IsNotExist(err) && create { + guid = newGUID() + if guid == "" { + return errors.New("maildir: failed to generate mailbox guid") + } + if err := d.storage.writeRegistry(d.name, guid); err != nil { + return err + } + } else { + return err + } + } + d.guid = guid + d.prefix = joinKey(d.storage.mboxRootPrefix(), guid) + return nil +} + +func (d *Dir) Init() error { + if err := d.ensurePrefix(true); err != nil { + return err + } + key := joinKey(d.prefix, markerName) + _, err := d.storage.client.PutObject(context.Background(), d.storage.bucket, key, bytes.NewReader(nil), 0, minio.PutObjectOptions{}) + if err == nil { + d.storage.invalidateList(d.prefix) + } + return err +} + +func (d *Dir) Exists() (bool, error) { + _, err := d.storage.readRegistry(d.name) + if err != nil { + if maildir.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (d *Dir) Children() ([]maildir.Dir, error) { + names, err := d.storage.ListDirs() + if err != nil { + return nil, err + } + var children []maildir.Dir + baseName := normalizeMailboxName(d.name) + prefix := baseName + hierarchySep + if strings.EqualFold(d.name, inboxName) { + prefix = "" + } + for _, name := range names { + if strings.EqualFold(name, baseName) { + continue + } + if prefix != "" { + if !strings.HasPrefix(name, prefix) { + continue + } + rel := strings.TrimPrefix(name, prefix) + if strings.Contains(rel, hierarchySep) { + continue + } + } else if strings.Contains(name, hierarchySep) { + continue + } + childDir, err := d.storage.Dir(name) + if err != nil { + continue + } + children = append(children, childDir) + } + return children, nil +} + +func (d *Dir) Remove(childExists bool) error { + if err := d.ensurePrefix(false); err != nil { + return err + } + if childExists { + for _, folder := range []string{"cur", "new"} { + prefix := joinKey(d.prefix, folder) + if err := d.deletePointers(prefix); err != nil { + return err + } + } + return nil + } + if err := d.deletePointers(joinKey(d.prefix, "cur")); err != nil { + return err + } + if err := d.deletePointers(joinKey(d.prefix, "new")); err != nil { + return err + } + if err := d.removePrefix(d.prefix); err != nil { + return err + } + return d.storage.deleteRegistry(d.name) +} + +func (d *Dir) removePrefix(prefix string) error { + keys, err := d.storage.listKeys(prefix) + if err != nil { + if isNotFound(err) { + return maildir.ErrNotExist + } + return err + } + err = d.storage.removeKeys(keys) + if err == nil { + d.storage.invalidateList(prefix) + } + return err +} + +func (d *Dir) deletePointers(prefix string) error { + keys, err := d.storage.listKeys(prefix) + if err != nil { + if isNotFound(err) { + return nil + } + return err + } + deleteKeys := make([]string, 0, len(keys)*2) + for _, key := range keys { + guid, _ := parsePointerKey(key) + if guid == "" { + continue + } + deleteKeys = append(deleteKeys, key) + deleteKeys = append(deleteKeys, joinKey(d.storage.blobRootPrefix(), guid)) + } + err = d.storage.removeKeys(deleteKeys) + if err == nil { + d.storage.invalidateList(prefix) + } + return err +} + +func (d *Dir) Rename(name string) error { + fullName := d.renameTarget(name) + if fullName == "" { + return errors.New("illegal mailbox name") + } + baseName := normalizeMailboxName(d.name) + guid, err := d.storage.readRegistry(d.name) + if err != nil { + return err + } + if _, err := d.storage.readRegistry(fullName); err == nil { + return errors.New("maildir: mailbox already exists") + } else if !maildir.IsNotExist(err) { + return err + } + names, err := d.storage.listRegistryNames() + if err != nil { + return err + } + for _, name := range names { + if !strings.HasPrefix(name, baseName+hierarchySep) { + continue + } + suffix := strings.TrimPrefix(name, baseName+hierarchySep) + if suffix == "" { + continue + } + childGUID, err := d.storage.readRegistry(name) + if err != nil { + return err + } + newChildName := fullName + hierarchySep + suffix + if err := d.storage.writeRegistry(newChildName, childGUID); err != nil { + return err + } + if err := d.storage.deleteRegistry(name); err != nil { + return err + } + } + if err := d.storage.writeRegistry(fullName, guid); err != nil { + return err + } + if err := d.storage.deleteRegistry(d.name); err != nil { + return err + } + d.name = fullName + d.guid = guid + d.prefix = joinKey(d.storage.mboxRootPrefix(), guid) + return nil +} + +func (d *Dir) LoadMetadata(state *maildir.Metadata) error { + if err := d.ensurePrefix(false); err != nil { + return err + } + store := newMetadataStore(d.storage, d.prefix) + return store.Load(state) +} + +func (d *Dir) WriteMetadata(state *maildir.Metadata) error { + if err := d.ensurePrefix(false); err != nil { + return err + } + store := newMetadataStore(d.storage, d.prefix) + return store.Write(state) +} + +func (d *Dir) Unseen() ([]maildir.Message, error) { + if err := d.ensurePrefix(false); err != nil { + return nil, err + } + keys, err := d.storage.listKeys(joinKey(d.prefix, "new")) + if err != nil { + if isNotFound(err) { + return nil, maildir.ErrNotExist + } + return nil, err + } + msgs := make([]maildir.Message, 0, len(keys)) + for _, key := range keys { + guid, flags := parsePointerKey(key) + msgs = append(msgs, &message{ + storage: d.storage, + bucket: d.storage.bucket, + pointerKey: key, + guid: guid, + flags: flags, + isNew: true, + }) + } + return msgs, nil +} + +func (d *Dir) Messages() ([]maildir.Message, error) { + if err := d.ensurePrefix(false); err != nil { + return nil, err + } + keys, err := d.storage.listKeys(d.prefix) + if err != nil { + if isNotFound(err) { + return nil, maildir.ErrNotExist + } + return nil, err + } + var msgs []maildir.Message + for _, key := range keys { + if !strings.Contains(key, "/cur/") && !strings.Contains(key, "/new/") { + continue + } + guid, flags := parsePointerKey(key) + msgs = append(msgs, &message{ + storage: d.storage, + bucket: d.storage.bucket, + pointerKey: key, + guid: guid, + flags: flags, + isNew: strings.Contains(key, "/new/"), + }) + } + return msgs, nil +} + +func (d *Dir) Create(flags []maildir.Flag) (maildir.Message, io.WriteCloser, error) { + if err := d.ensurePrefix(false); err != nil { + return nil, nil, err + } + guid := newGUID() + file, err := os.CreateTemp("", "maildir-s3-blob-*") + if err != nil { + return nil, nil, err + } + + msg := &message{ + storage: d.storage, + bucket: d.storage.bucket, + pointerKey: d.pointerKey(guid, flags, len(flags) == 0), + guid: guid, + flags: flags, + isNew: len(flags) == 0, + } + return msg, &blobWriter{file: file, msg: msg}, nil +} + +func (d *Dir) NewDelivery() (maildir.Delivery, error) { + if err := d.ensurePrefix(false); err != nil { + return nil, err + } + guid := newGUID() + file, err := os.CreateTemp("", "maildir-s3-delivery-*") + if err != nil { + return nil, err + } + return &delivery{dir: d, file: file, guid: guid}, nil +} + +func (d *Dir) pointerKey(guid string, flags []maildir.Flag, isNew bool) string { + name := pointerName(guid, flags, isNew) + folder := "cur" + if isNew && len(flags) == 0 { + folder = "new" + } + return joinKey(d.prefix, folder, name) +} + +func (d *Dir) renameTarget(name string) string { + if strings.EqualFold(d.name, inboxName) { + return name + } + if d.name == "" { + return name + } + parts := strings.Split(d.name, hierarchySep) + if len(parts) <= 1 { + return name + } + parent := strings.Join(parts[:len(parts)-1], hierarchySep) + if parent == "" { + return name + } + return parent + hierarchySep + name +} + +type blobWriter struct { + file *os.File + msg *message +} + +func (w *blobWriter) Write(p []byte) (int, error) { + return w.file.Write(p) +} + +func (w *blobWriter) Close() error { + info, err := w.file.Stat() + if err != nil { + _ = w.file.Close() + _ = os.Remove(w.file.Name()) + return err + } + if _, err := w.file.Seek(0, io.SeekStart); err != nil { + _ = w.file.Close() + _ = os.Remove(w.file.Name()) + return err + } + blobKey := joinKey(w.msg.storage.blobRootPrefix(), w.msg.guid) + _, err = w.msg.storage.client.PutObject(context.Background(), w.msg.bucket, blobKey, w.file, info.Size(), minio.PutObjectOptions{}) + if err != nil { + _ = w.file.Close() + _ = os.Remove(w.file.Name()) + return err + } + if err := w.file.Close(); err != nil { + _ = os.Remove(w.file.Name()) + return err + } + if err := os.Remove(w.file.Name()); err != nil { + return err + } + return w.msg.createPointer() +} + +type delivery struct { + dir *Dir + file *os.File + guid string +} + +func (d *delivery) Write(p []byte) (int, error) { + return d.file.Write(p) +} + +func (d *delivery) Close() error { + info, err := d.file.Stat() + if err != nil { + _ = d.file.Close() + _ = os.Remove(d.file.Name()) + return err + } + if _, err := d.file.Seek(0, io.SeekStart); err != nil { + _ = d.file.Close() + _ = os.Remove(d.file.Name()) + return err + } + blobKey := joinKey(d.dir.storage.blobRootPrefix(), d.guid) + _, err = d.dir.storage.client.PutObject(context.Background(), d.dir.storage.bucket, blobKey, d.file, info.Size(), minio.PutObjectOptions{}) + if err != nil { + _ = d.file.Close() + _ = os.Remove(d.file.Name()) + return err + } + if err := d.file.Close(); err != nil { + _ = os.Remove(d.file.Name()) + return err + } + if err := os.Remove(d.file.Name()); err != nil { + return err + } + key := d.dir.pointerKey(d.guid, nil, true) + _, err = d.dir.storage.client.PutObject(context.Background(), d.dir.storage.bucket, key, bytes.NewReader(nil), 0, minio.PutObjectOptions{}) + if err == nil { + d.dir.storage.invalidateListForKey(key) + } + return err +} + +func (d *delivery) Abort() error { + name := d.file.Name() + if err := d.file.Close(); err != nil { + _ = os.Remove(name) + return err + } + return os.Remove(name) +} + +type message struct { + storage *Storage + bucket string + pointerKey string + guid string + flags []maildir.Flag + isNew bool +} + +func (m *message) Key() string { + return m.guid +} + +func (m *message) Name() string { + return path.Base(m.pointerKey) +} + +func (m *message) Flags() []maildir.Flag { + return append([]maildir.Flag{}, m.flags...) +} + +func (m *message) SetFlags(flags []maildir.Flag) error { + newKey := m.pointerKeyForFlags(flags) + if newKey == m.pointerKey { + m.flags = append([]maildir.Flag{}, flags...) + if m.isNew { + m.isNew = false + } + return nil + } + if err := m.createPointerWithKey(newKey); err != nil { + return err + } + if err := m.storage.client.RemoveObject(context.Background(), m.bucket, m.pointerKey, minio.RemoveObjectOptions{}); err != nil { + return err + } + m.storage.invalidateListForKey(m.pointerKey) + m.pointerKey = newKey + m.flags = append([]maildir.Flag{}, flags...) + m.isNew = false + return nil +} + +func (m *message) Open() (io.ReadCloser, error) { + blobKey := joinKey(m.storage.blobRootPrefix(), m.guid) + if m.storage.cache != nil { + if data, ok := m.storage.cache.Get(blobKey); ok { + return io.NopCloser(bytes.NewReader(data)), nil + } + } + obj, err := m.storage.client.GetObject(context.Background(), m.bucket, blobKey, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + if m.storage.cache == nil || m.storage.maxCacheItemSize <= 0 { + return obj, nil + } + info, err := obj.Stat() + if err != nil { + _ = obj.Close() + if isNotFound(err) { + return nil, maildir.ErrNotExist + } + return nil, err + } + if info.Size <= 0 || info.Size > m.storage.maxCacheItemSize { + return obj, nil + } + data, err := io.ReadAll(obj) + if err != nil { + _ = obj.Close() + return nil, err + } + _ = obj.Close() + m.storage.cache.Set(blobKey, data, int64(len(data))) + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (m *message) Stat() (maildir.Info, error) { + blobKey := joinKey(m.storage.blobRootPrefix(), m.guid) + stat, err := m.storage.client.StatObject(context.Background(), m.bucket, blobKey, minio.StatObjectOptions{}) + if err != nil { + if isNotFound(err) { + return nil, maildir.ErrNotExist + } + return nil, err + } + return &info{size: stat.Size, modTime: stat.LastModified}, nil +} + +func (m *message) Chtimes(atime, mtime time.Time) error { + return nil +} + +func (m *message) Remove() error { + if err := m.storage.client.RemoveObject(context.Background(), m.bucket, m.pointerKey, minio.RemoveObjectOptions{}); err != nil { + return err + } + m.storage.invalidateListForKey(m.pointerKey) + blobKey := joinKey(m.storage.blobRootPrefix(), m.guid) + if err := m.storage.client.RemoveObject(context.Background(), m.bucket, blobKey, minio.RemoveObjectOptions{}); err != nil { + return err + } + return nil +} + +func (m *message) MoveTo(target maildir.Dir) error { + other, ok := target.(*Dir) + if !ok { + return errors.New("maildir: unsupported target dir implementation") + } + newKey := other.pointerKey(m.guid, m.flags, m.isNew) + if err := m.createPointerWithKey(newKey); err != nil { + return err + } + if err := m.storage.client.RemoveObject(context.Background(), m.bucket, m.pointerKey, minio.RemoveObjectOptions{}); err != nil { + return err + } + m.storage.invalidateListForKey(m.pointerKey) + m.pointerKey = newKey + return nil +} + +func (m *message) CopyTo(target maildir.Dir) (maildir.Message, error) { + other, ok := target.(*Dir) + if !ok { + return nil, errors.New("maildir: unsupported target dir implementation") + } + newGUID := newGUID() + src := minio.CopySrcOptions{Bucket: m.bucket, Object: joinKey(m.storage.blobRootPrefix(), m.guid)} + dst := minio.CopyDestOptions{Bucket: m.bucket, Object: joinKey(m.storage.blobRootPrefix(), newGUID)} + if _, err := m.storage.client.CopyObject(context.Background(), dst, src); err != nil { + return nil, err + } + newMsg := &message{ + storage: m.storage, + bucket: m.bucket, + guid: newGUID, + flags: append([]maildir.Flag{}, m.flags...), + isNew: m.isNew, + pointerKey: other.pointerKey(newGUID, m.flags, m.isNew), + } + if err := newMsg.createPointer(); err != nil { + return nil, err + } + return newMsg, nil +} + +func (m *message) pointerKeyForFlags(flags []maildir.Flag) string { + parts := strings.Split(m.pointerKey, "/") + if len(parts) < 2 { + return m.pointerKey + } + base := pointerName(m.guid, flags, false) + parts[len(parts)-1] = base + if m.isNew && len(parts) >= 2 { + parts[len(parts)-2] = "cur" + } + return strings.Join(parts, "/") +} + +func (m *message) createPointer() error { + return m.createPointerWithKey(m.pointerKey) +} + +func (m *message) createPointerWithKey(key string) error { + _, err := m.storage.client.PutObject(context.Background(), m.bucket, key, bytes.NewReader(nil), 0, minio.PutObjectOptions{}) + if err == nil { + m.storage.invalidateListForKey(key) + } + return err +} + +type info struct { + size int64 + modTime time.Time +} + +func (i *info) Size() int64 { + return i.size +} + +func (i *info) ModTime() time.Time { + return i.modTime +} + +func parsePointerKey(key string) (string, []maildir.Flag) { + name := path.Base(key) + parts := strings.SplitN(name, ":2,", 2) + guid := parts[0] + if len(parts) < 2 { + return guid, nil + } + var flags []maildir.Flag + for _, r := range parts[1] { + flags = append(flags, maildir.Flag(r)) + } + return guid, flags +} + +func pointerName(guid string, flags []maildir.Flag, isNew bool) string { + if isNew && len(flags) == 0 { + return guid + } + flagRunes := make([]rune, 0, len(flags)) + for _, flag := range flags { + flagRunes = append(flagRunes, rune(flag)) + } + slices.Sort(flagRunes) + return guid + ":2," + string(flagRunes) +} + +func joinKey(parts ...string) string { + var clean []string + for _, part := range parts { + if part == "" { + continue + } + clean = append(clean, strings.Trim(part, "/")) + } + if len(clean) == 0 { + return "" + } + return path.Join(clean...) +} + +func normalizeMailboxName(name string) string { + if name == "" { + return "" + } + parts := strings.Split(name, hierarchySep) + normalized := make([]string, 0, len(parts)) + for i, part := range parts { + if i == 0 && strings.EqualFold(part, inboxName) { + continue + } + if part == "" { + continue + } + normalized = append(normalized, part) + } + if len(normalized) == 0 { + return inboxName + } + return strings.Join(normalized, hierarchySep) +} + +func newGUID() string { + buf := make([]byte, 16) + if _, err := rand.Read(buf); err != nil { + return fmt.Sprintf("%x", time.Now().UnixNano()) + } + return strings.ToLower(hexString(buf)) +} + +func hexString(buf []byte) string { + const hextable = "0123456789abcdef" + out := make([]byte, len(buf)*2) + for i, b := range buf { + out[i*2] = hextable[b>>4] + out[i*2+1] = hextable[b&0x0f] + } + return string(out) +} + +func validMboxPart(name string) bool { + if strings.ContainsAny(name, ":*?\"<>|") { + return false + } + for _, ch := range name { + if ch < ' ' { + return false + } + } + return !strings.Contains(name, "..") +} + +func isNotFound(err error) bool { + resp := minio.ToErrorResponse(err) + if resp.Code == "NoSuchKey" || resp.Code == "NoSuchBucket" || resp.Code == "NotFound" { + return true + } + return resp.StatusCode == 404 +} diff --git a/maildir/s3/s3_test.go b/maildir/s3/s3_test.go new file mode 100644 index 0000000..bf84adc --- /dev/null +++ b/maildir/s3/s3_test.go @@ -0,0 +1,208 @@ +package s3 + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/foxcpp/go-imap-maildir/maildir" + "github.com/foxcpp/go-imap-maildir/maildir/s3/testing" +) + +func TestS3ListDirs(t *testing.T) { + storage, cleanup := newTestStorage(t) + defer cleanup() + + inbox, err := storage.Dir("INBOX") + if err != nil { + t.Fatal(err) + } + if err := inbox.Init(); err != nil { + t.Fatal(err) + } + + projects, err := storage.Dir("Projects") + if err != nil { + t.Fatal(err) + } + if err := projects.Init(); err != nil { + t.Fatal(err) + } + + list, err := storage.ListDirs() + if err != nil { + t.Fatal(err) + } + if len(list) != 1 || list[0] != "Projects" { + t.Fatalf("expected only Projects, got %v", list) + } +} + +func TestS3CopyMove(t *testing.T) { + storage, cleanup := newTestStorage(t) + defer cleanup() + + inbox := mustInitDir(t, storage, "INBOX") + archive := mustInitDir(t, storage, "Archive") + + msg := createMessage(t, inbox, []byte("hello"), nil) + + copied, err := msg.CopyTo(archive) + if err != nil { + t.Fatal(err) + } + if copied.Key() == msg.Key() { + t.Fatalf("expected copy to generate new key") + } + if readMessage(t, copied) != "hello" { + t.Fatalf("expected copied message content") + } + + if err := msg.MoveTo(archive); err != nil { + t.Fatal(err) + } + msgs, err := archive.Messages() + if err != nil { + t.Fatal(err) + } + if len(msgs) < 2 { + t.Fatalf("expected at least 2 messages after copy+move, got %d", len(msgs)) + } +} + +func TestS3RenameNoCopy(t *testing.T) { + storage, cleanup := newTestStorage(t) + defer cleanup() + + oldDir := mustInitDir(t, storage, "Old") + createMessage(t, oldDir, []byte("payload"), nil) + + if err := oldDir.Rename("New"); err != nil { + t.Fatal(err) + } + + oldDir = mustDir(t, storage, "Old") + exists, err := oldDir.Exists() + if err != nil { + t.Fatal(err) + } + if exists { + t.Fatalf("expected Old to be invalid after rename") + } + + newDir := mustDir(t, storage, "New") + msgs, err := newDir.Messages() + if err != nil { + t.Fatal(err) + } + if len(msgs) != 1 { + t.Fatalf("expected 1 message in New, got %d", len(msgs)) + } +} + +func TestS3Delivery(t *testing.T) { + storage, cleanup := newTestStorage(t) + defer cleanup() + + inbox := mustInitDir(t, storage, "INBOX") + delivery, err := inbox.NewDelivery() + if err != nil { + t.Fatal(err) + } + if _, err := delivery.Write([]byte("hello")); err != nil { + _ = delivery.Abort() + t.Fatal(err) + } + if err := delivery.Close(); err != nil { + t.Fatal(err) + } + + msgs, err := inbox.Messages() + if err != nil { + t.Fatal(err) + } + if len(msgs) != 1 { + t.Fatalf("expected 1 message, got %d", len(msgs)) + } + if readMessage(t, msgs[0]) != "hello" { + t.Fatalf("expected delivered message content") + } +} + +func TestS3DeliveryAbort(t *testing.T) { + storage, cleanup := newTestStorage(t) + defer cleanup() + + inbox := mustInitDir(t, storage, "INBOX") + delivery, err := inbox.NewDelivery() + if err != nil { + t.Fatal(err) + } + if _, err := delivery.Write([]byte("discard")); err != nil { + _ = delivery.Abort() + t.Fatal(err) + } + if err := delivery.Abort(); err != nil { + t.Fatal(err) + } + + msgs, err := inbox.Messages() + if err != nil { + t.Fatal(err) + } + if len(msgs) != 0 { + t.Fatalf("expected 0 messages, got %d", len(msgs)) + } +} + +func newTestStorage(t *testing.T) (*Storage, func()) { + client, bucket, rootPrefix, cleanup := s3testing.StartMinioShared(t) + basePath := "test-" + time.Now().UTC().Format("20060102150405.000000000") + storage := NewProvider(client, bucket, rootPrefix).Storage(basePath).(*Storage) + return storage, cleanup +} + +func mustInitDir(t *testing.T, storage *Storage, name string) maildir.Dir { + dir := mustDir(t, storage, name) + if err := dir.Init(); err != nil { + t.Fatal(err) + } + return dir +} + +func mustDir(t *testing.T, storage *Storage, name string) maildir.Dir { + dir, err := storage.Dir(name) + if err != nil { + t.Fatal(err) + } + return dir +} + +func createMessage(t *testing.T, dir maildir.Dir, body []byte, flags []maildir.Flag) maildir.Message { + msg, writer, err := dir.Create(flags) + if err != nil { + t.Fatal(err) + } + if _, err := writer.Write(body); err != nil { + _ = writer.Close() + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return msg +} + +func readMessage(t *testing.T, msg maildir.Message) string { + r, err := msg.Open() + if err != nil { + t.Fatal(err) + } + defer r.Close() + buf := &bytes.Buffer{} + if _, err := buf.ReadFrom(r); err != nil { + t.Fatal(err) + } + return strings.TrimSpace(buf.String()) +} diff --git a/maildir/s3/testing/minio.go b/maildir/s3/testing/minio.go new file mode 100644 index 0000000..795692d --- /dev/null +++ b/maildir/s3/testing/minio.go @@ -0,0 +1,301 @@ +package s3testing + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +func StartMinio(t *testing.T) (*minio.Client, string, string, func()) { + t.Helper() + bin, binCleanup := ensureMinioBinary(t) + accessKey := "minio" + secretKey := "minio123" + dataDir := t.TempDir() + addr := freeLocalAddr(t) + consoleAddr := freeLocalAddr(t) + + ctx, cancel := context.WithCancel(context.Background()) + cmd := exec.CommandContext(ctx, bin, "server", dataDir, "--address", addr, "--console-address", consoleAddr) + cmd.Env = append(os.Environ(), + "MINIO_ROOT_USER="+accessKey, + "MINIO_ROOT_PASSWORD="+secretKey, + ) + cmd.Stdout = &bytes.Buffer{} + cmd.Stderr = &bytes.Buffer{} + if err := cmd.Start(); err != nil { + binCleanup() + t.Fatalf("start minio: %v", err) + } + + if err := waitForMinioReady(addr, 30*time.Second); err != nil { + cancel() + _ = cmd.Wait() + binCleanup() + t.Fatalf("minio not ready: %v", err) + } + + client, err := minio.New(addr, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: false, + }) + if err != nil { + cancel() + _ = cmd.Wait() + binCleanup() + t.Fatalf("minio client: %v", err) + } + + bucket := "test-bucket" + if err := client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil { + exists, errExists := client.BucketExists(context.Background(), bucket) + if errExists != nil || !exists { + cancel() + _ = cmd.Wait() + binCleanup() + t.Fatalf("make bucket: %v", err) + } + } + + rootPrefix := "root" + cleanup := func() { + RemoveAllObjects(t, client, bucket, rootPrefix) + cancel() + _ = cmd.Wait() + binCleanup() + } + return client, bucket, rootPrefix, cleanup +} + +var sharedMinio struct { + mu sync.Mutex + started bool + refcount int + client *minio.Client + bucket string + dataDir string + cancel context.CancelFunc + cmd *exec.Cmd + binCleanup func() +} + +var sharedMinioCounter uint64 + +func StartMinioShared(t *testing.T) (*minio.Client, string, string, func()) { + t.Helper() + sharedMinio.mu.Lock() + if !sharedMinio.started { + bin, binCleanup := ensureMinioBinary(t) + accessKey := "minio" + secretKey := "minio123" + dataDir, err := os.MkdirTemp("", "go-imap-maildir-minio-") + if err != nil { + sharedMinio.mu.Unlock() + t.Fatalf("create minio data dir: %v", err) + } + addr := freeLocalAddr(t) + consoleAddr := freeLocalAddr(t) + + ctx, cancel := context.WithCancel(context.Background()) + cmd := exec.CommandContext(ctx, bin, "server", dataDir, "--address", addr, "--console-address", consoleAddr) + cmd.Env = append(os.Environ(), + "MINIO_ROOT_USER="+accessKey, + "MINIO_ROOT_PASSWORD="+secretKey, + ) + cmd.Stdout = &bytes.Buffer{} + cmd.Stderr = &bytes.Buffer{} + if err := cmd.Start(); err != nil { + sharedMinio.mu.Unlock() + binCleanup() + _ = os.RemoveAll(dataDir) + t.Fatalf("start minio: %v", err) + } + + if err := waitForMinioReady(addr, 30*time.Second); err != nil { + cancel() + _ = cmd.Wait() + binCleanup() + _ = os.RemoveAll(dataDir) + sharedMinio.mu.Unlock() + t.Fatalf("minio not ready: %v", err) + } + + client, err := minio.New(addr, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: false, + }) + if err != nil { + cancel() + _ = cmd.Wait() + binCleanup() + _ = os.RemoveAll(dataDir) + sharedMinio.mu.Unlock() + t.Fatalf("minio client: %v", err) + } + + bucket := "test-bucket" + if err := client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil { + exists, errExists := client.BucketExists(context.Background(), bucket) + if errExists != nil || !exists { + cancel() + _ = cmd.Wait() + binCleanup() + _ = os.RemoveAll(dataDir) + sharedMinio.mu.Unlock() + t.Fatalf("make bucket: %v", err) + } + } + + sharedMinio.started = true + sharedMinio.client = client + sharedMinio.bucket = bucket + sharedMinio.dataDir = dataDir + sharedMinio.cancel = cancel + sharedMinio.cmd = cmd + sharedMinio.binCleanup = binCleanup + } + sharedMinio.refcount++ + client := sharedMinio.client + bucket := sharedMinio.bucket + sharedMinio.mu.Unlock() + + prefixID := atomic.AddUint64(&sharedMinioCounter, 1) + rootPrefix := "root-" + strconv.FormatUint(prefixID, 10) + "-" + time.Now().UTC().Format("20060102150405.000000000") + cleanup := func() { + RemoveAllObjects(t, client, bucket, rootPrefix) + sharedMinio.mu.Lock() + sharedMinio.refcount-- + if sharedMinio.refcount == 0 && sharedMinio.started { + sharedMinio.cancel() + _ = sharedMinio.cmd.Wait() + sharedMinio.binCleanup() + _ = os.RemoveAll(sharedMinio.dataDir) + sharedMinio.started = false + sharedMinio.client = nil + sharedMinio.bucket = "" + sharedMinio.dataDir = "" + sharedMinio.cancel = nil + sharedMinio.cmd = nil + sharedMinio.binCleanup = nil + } + sharedMinio.mu.Unlock() + } + return client, bucket, rootPrefix, cleanup +} + +func RemoveAllObjects(t *testing.T, client *minio.Client, bucket, prefix string) { + t.Helper() + if prefix == "" { + return + } + for obj := range client.ListObjects(context.Background(), bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if obj.Err != nil { + t.Fatalf("list objects: %v", obj.Err) + } + if err := client.RemoveObject(context.Background(), bucket, obj.Key, minio.RemoveObjectOptions{}); err != nil { + t.Fatalf("remove object: %v", err) + } + } +} + +func ensureMinioBinary(t *testing.T) (string, func()) { + if bin := os.Getenv("MINIO_BIN"); bin != "" { + return bin, func() {} + } + url, ok := minioBinaryURL() + if !ok { + t.Fatalf("unsupported platform for minio binary") + } + cacheDir := filepath.Join(os.TempDir(), "go-imap-maildir-minio") + if err := os.MkdirAll(cacheDir, 0700); err != nil { + t.Fatalf("create cache dir: %v", err) + } + binPath := filepath.Join(cacheDir, "minio") + if _, err := os.Stat(binPath); err == nil { + return binPath, func() {} + } + resp, err := http.Get(url) + if err != nil { + t.Fatalf("download minio: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("download minio: %s", resp.Status) + } + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0700) + if err != nil { + t.Fatalf("write minio: %v", err) + } + if _, err := io.Copy(file, resp.Body); err != nil { + _ = file.Close() + _ = os.Remove(binPath) + t.Fatalf("write minio: %v", err) + } + if err := file.Close(); err != nil { + _ = os.Remove(binPath) + t.Fatalf("close minio: %v", err) + } + return binPath, func() {} +} + +func minioBinaryURL() (string, bool) { + osName := runtime.GOOS + arch := runtime.GOARCH + switch osName { + case "darwin", "linux": + // ok + default: + return "", false + } + switch arch { + case "amd64", "arm64": + // ok + default: + return "", false + } + return "https://dl.min.io/server/minio/release/" + osName + "-" + arch + "/minio", true +} + +func freeLocalAddr(t *testing.T) string { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + addr := listener.Addr().String() + if err := listener.Close(); err != nil { + t.Fatalf("close listener: %v", err) + } + return addr +} + +func waitForMinioReady(addr string, timeout time.Duration) error { + url := "http://" + addr + "/minio/health/ready" + deadline := time.Now().Add(timeout) + client := &http.Client{Timeout: 2 * time.Second} + for time.Now().Before(deadline) { + resp, err := client.Get(url) + if err == nil { + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return nil + } + } + time.Sleep(200 * time.Millisecond) + } + return errors.New("timeout") +} diff --git a/maildird/main.go b/maildird/main.go index c74caac..9cfbfe1 100644 --- a/maildird/main.go +++ b/maildird/main.go @@ -8,7 +8,9 @@ import ( "os/signal" "github.com/emersion/go-imap/server" + "github.com/foxcpp/go-imap-maildir" + "github.com/foxcpp/go-imap-maildir/maildir/fs" ) func main() { @@ -21,7 +23,7 @@ func main() { pathTemplate := os.Args[1] endpoint := os.Args[2] - bkd, err := imapmaildir.New(pathTemplate) + bkd, err := imapmaildir.New(pathTemplate, fs.Provider{}, nil) bkd.Debug = log.New(os.Stderr, "imapmaildir[debug]: ", 0) defer bkd.Close() if err != nil { diff --git a/metadata.go b/metadata.go new file mode 100644 index 0000000..b9e3f19 --- /dev/null +++ b/metadata.go @@ -0,0 +1,16 @@ +package imapmaildir + +import ( + "github.com/foxcpp/go-imap-maildir/maildir" +) + +func (m *Mailbox) loadMetadataState() error { + if m.state.meta == nil { + m.state.meta = &maildir.Metadata{} + } + return m.dir.LoadMetadata(m.state.meta) +} + +func (m *Mailbox) writeMetadataState(state *maildir.Metadata) error { + return m.dir.WriteMetadata(state) +} diff --git a/user.go b/user.go index 823d2d5..f20e004 100644 --- a/user.go +++ b/user.go @@ -3,368 +3,837 @@ package imapmaildir import ( "errors" "fmt" - "os" - "path/filepath" + "io" "strings" + "sync" "time" - "github.com/asdine/storm/v3" + "github.com/emersion/go-imap" "github.com/emersion/go-imap/backend" - "github.com/emersion/go-maildir" - "go.etcd.io/bbolt" + + "github.com/foxcpp/go-imap-maildir/maildir" ) const ( InboxName = "INBOX" HierarchySep = "." MaxMboxNesting = 100 - - IndexFile = "imapmaildir-index.db" ) func validMboxPart(name string) bool { - // Restrict characters that may be problematic for FS handling. - - // This is list of characters not allowed in NTFS minus 0x00 (handled - // below), in Unix world many of these characters may be troublesome to - // handle in shell scripts. if strings.ContainsAny(name, ":*?\"<>|") { return false } - // Disallow ASCII control characters (including 0x00). for _, ch := range name { if ch < ' ' { return false } } - // Prevent directory structure escaping. return !strings.Contains(name, "..") } -type User struct { - b *Backend - - name string - basePath string -} - -func (u *User) prepareMboxPath(mbox string) (fsPath string, parts []string, err error) { +func (u *User) validateMboxName(mbox string) error { if strings.EqualFold(mbox, InboxName) { - return u.basePath, []string{}, nil + return nil } - - // Verify validity before attempting to do anything. + parts := strings.Split(mbox, HierarchySep) if len(parts) > MaxMboxNesting { - return "", nil, errors.New("mailbox nesting limit exceeded") + return errors.New("mailbox nesting limit exceeded") } - fsPath = u.basePath - nameParts := strings.Split(mbox, HierarchySep) - for i, part := range nameParts { + for i, part := range parts { + if i == 0 && strings.EqualFold(part, InboxName) { + continue + } if part == "" { - // Strip the possible trailing separator but not allow empty parts - // in general. if i != len(parts)-1 { - return "", nil, errors.New("illegal mailbox name") + return errors.New("illegal mailbox name") } continue } - if !validMboxPart(part) { u.b.Log.Printf("illegal mailbox name requested by %s: %v", u.name, mbox) - return "", nil, errors.New("illegal mailbox name") + return errors.New("illegal mailbox name") } - fsPath += string(filepath.Separator) + "." + part } - - return fsPath, parts, nil + return nil } -func (u *User) mboxName(fsPath string) (string, error) { - fsPath = strings.TrimPrefix(fsPath, u.basePath+string(filepath.Separator)) - if fsPath == "" { - return InboxName, nil +func mailboxAncestors(mbox string) []string { + if strings.EqualFold(mbox, InboxName) { + return []string{InboxName} + } + parts := strings.Split(mbox, HierarchySep) + ancestors := make([]string, 0, len(parts)) + current := "" + for i, part := range parts { + if i == 0 && strings.EqualFold(part, InboxName) { + continue + } + if part == "" { + continue + } + if current == "" { + current = part + } else { + current = current + HierarchySep + part + } + ancestors = append(ancestors, current) } + return ancestors +} - parts := strings.Split(fsPath, string(filepath.Separator)) - if len(parts) > MaxMboxNesting { - return "", errors.New("mailbox nesting limit exceeded") +func relativeMailboxName(existingName, newName string) (string, error) { + if strings.EqualFold(existingName, InboxName) { + return newName, nil + } + parts := strings.Split(existingName, HierarchySep) + if len(parts) <= 1 { + return newName, nil } + parent := strings.Join(parts[:len(parts)-1], HierarchySep) + if parent == "" { + return newName, nil + } + prefix := parent + HierarchySep + if !strings.HasPrefix(newName, prefix) { + return "", errors.New("illegal mailbox name") + } + rel := strings.TrimPrefix(newName, prefix) + if rel == "" { + return "", errors.New("illegal mailbox name") + } + return rel, nil +} - mboxParts := make([]string, 0, len(parts)) - for _, part := range parts { - if !strings.HasPrefix(part, ".") { - return "", fmt.Errorf("not a maildir++ path: %v", fsPath) - } +type User struct { + b *Backend - mboxParts = append(mboxParts, part[1:]) - } + name string + storage maildir.Storage + + mailboxesLock sync.Mutex + mailboxes map[string]*Mailbox - return strings.Join(mboxParts, HierarchySep), nil + limitLock sync.Mutex + appendLimit *uint32 + + mboxLimitLock sync.Mutex + mboxLimits map[string]*uint32 } func (u *User) Username() string { return u.name } -func (u *User) ListMailboxes(subscribed bool) ([]backend.Mailbox, error) { - // TODO: Figure out a fast way to filter subscribed/unsubscribed - // directories. +func (u *User) getMailbox(mbox string) (*Mailbox, bool) { + u.mailboxesLock.Lock() + defer u.mailboxesLock.Unlock() - mboxes := []backend.Mailbox{ - &Mailbox{ - // Inbox always exists. - name: InboxName, - path: u.basePath, - }, - } + mb, ok := u.mailboxes[mbox] + return mb, ok +} + +type DefaultMailboxSpec struct { + Name string + SpecialUse string +} - err := filepath.Walk(u.basePath, func(path string, info os.FileInfo, err error) error { +var defaultMailboxSpecs = []DefaultMailboxSpec{ + {Name: "Trash", SpecialUse: imap.TrashAttr}, + {Name: "Junk", SpecialUse: imap.JunkAttr}, + {Name: "Sent", SpecialUse: imap.SentAttr}, + {Name: "Archive", SpecialUse: imap.ArchiveAttr}, + {Name: "Draft", SpecialUse: imap.DraftsAttr}, +} + +func (u *User) ensureDefaultMailboxes() { + specs := u.b.DefaultMailboxes + if len(specs) == 0 { + specs = defaultMailboxSpecs + } + for _, spec := range specs { + name := spec.Name + mboxDir, err := u.storage.Dir(name) if err != nil { - // Ignore errors, return as much as possible. - u.b.Log.Printf("error during mailboxes iteration: %v", err) - return nil + continue } - if !info.IsDir() { - return nil + exists, err := mboxDir.Exists() + if err != nil { + continue } - - // Inbox is already added explicitly above. - if path == u.basePath { - return nil + if !exists { + _ = u.CreateMailbox(name) } - - if !strings.HasPrefix(info.Name(), ".") { - return filepath.SkipDir + u.mailboxesLock.Lock() + mbox := u.ensureMailbox(name) + u.mailboxesLock.Unlock() + if spec.SpecialUse != "" { + _ = u.setMailboxSpecialUse(mbox, spec.SpecialUse) } + } +} - mboxName, err := u.mboxName(path) - if err != nil { - u.b.Log.Printf("error during mailboxes iteration: %v", err) - return filepath.SkipDir +func defaultMailboxSpecialUse(name string) (string, bool) { + for _, spec := range defaultMailboxSpecs { + if strings.EqualFold(spec.Name, name) { + return spec.SpecialUse, true } + } + return "", false +} + +func (u *User) setMailboxSpecialUse(mbox *Mailbox, value string) error { + if mbox == nil { + return nil + } + if err := mbox.loadMetadataState(); err != nil { + return err + } + mbox.state.meta.EnsureGUID() + if err := mbox.writeMetadataState(mbox.state.meta); err != nil { + return err + } + key := "priv/" + mbox.state.meta.GUID + "/specialuse" + attrsStore, err := u.storage.Attributes() + if err != nil { + return err + } + attrs, err := attrsStore.Read() + if err != nil { + return err + } + if value == "" { + delete(attrs, key) + } else { + attrs[key] = value + } + return attrsStore.Write(attrs) +} + +func (u *User) mailboxSpecialUse(mbox *Mailbox) ([]string, error) { + if mbox == nil { + return nil, nil + } + if err := mbox.loadMetadataState(); err != nil { + return nil, err + } + guid := mbox.state.meta.GUID + if guid == "" { + return nil, nil + } + attrsStore, err := u.storage.Attributes() + if err != nil { + return nil, err + } + attrs, err := attrsStore.Read() + if err != nil { + return nil, err + } + value := attrs["priv/"+guid+"/specialuse"] + if value == "" { + return nil, nil + } + return strings.Fields(value), nil +} - u.b.Debug.Printf("listing mbox (%v, %v)", mboxName, path) +func (u *User) ensureMailbox(mbox string) *Mailbox { + if m, ok := u.mailboxes[mbox]; ok { + return m + } - // Note that Mailbox object has nil handle. - mboxes = append(mboxes, &Mailbox{ - b: u.b, - username: u.name, - name: mboxName, - path: path, - }) + mboxDir, err := u.storage.Dir(mbox) + if err != nil { return nil - }) + } + + mboxObj := &Mailbox{ + b: u.b, + user: u, + name: mbox, + dir: mboxDir, + state: u.b.getMailboxState(u.name, mbox), + subscribed: true, + } + if u.mailboxes == nil { + u.mailboxes = map[string]*Mailbox{} + } + u.mailboxes[mbox] = mboxObj + return mboxObj +} + +func (u *User) ListMailboxes(subscribed bool) ([]imap.MailboxInfo, error) { + u.mailboxesLock.Lock() + defer u.mailboxesLock.Unlock() + + if u.mailboxes == nil { + u.mailboxes = map[string]*Mailbox{} + } + + var mboxes []imap.MailboxInfo + + inbox := u.ensureMailbox(InboxName) + if !subscribed || inbox.subscribed { + info, err := inbox.Info() + if err == nil { + mboxes = append(mboxes, *info) + } + } + + names, err := u.storage.ListDirs() if err != nil { u.b.Log.Printf("failed to list mailboxes: %v", err) - return nil, errors.New("I/O error") + return nil, fmt.Errorf("I/O error: %w", err) + } + for _, name := range names { + mbox := u.ensureMailbox(name) + if mbox == nil { + continue + } + if subscribed && !mbox.subscribed { + continue + } + mboxInfo, err := mbox.Info() + if err != nil { + continue + } + mboxes = append(mboxes, *mboxInfo) } return mboxes, nil } -func (u *User) openDB(fsPath, mbox string) (*storm.DB, error) { - u.b.dbsLock.Lock() - defer u.b.dbsLock.Unlock() +func (u *User) GetMailbox(name string, readOnly bool, conn backend.Conn) (*imap.MailboxStatus, backend.Mailbox, error) { + if err := u.validateMboxName(name); err != nil { + return nil, nil, err + } + mboxDir, err := u.storage.Dir(name) + if err != nil { + return nil, nil, err + } + exists, err := mboxDir.Exists() + if err != nil { + return nil, nil, fmt.Errorf("I/O error: %w", err) + } + if !exists { + return nil, nil, backend.ErrNoSuchMailbox + } - key := u.name + "\x00" + mbox - handle, ok := u.b.dbs[key] - if ok { - handle.uses++ - u.b.Debug.Printf("%d uses for %s/%s mbox", handle.uses, u.name, mbox) - u.b.dbs[key] = handle - db := handle.db - return db, nil + mbox := u.ensureMailbox(name) + status, err := u.Status(name, []imap.StatusItem{ + imap.StatusMessages, imap.StatusRecent, imap.StatusUnseen, + imap.StatusUidNext, imap.StatusUidValidity, + }) + if err != nil { + return nil, nil, err } - db, err := storm.Open(filepath.Join(fsPath, IndexFile)) + entries, err := mbox.listEntries() if err != nil { - return nil, err + return nil, nil, fmt.Errorf("I/O error: %w", err) } - u.b.dbs[key] = mailboxHandle{ - uses: 1, - db: db, + var uids []uint32 + var recent imap.SeqSet + for _, entry := range entries { + uids = append(uids, entry.uid) + if entry.meta.recent { + recent.AddNum(entry.uid) + entry.meta.recent = false + } } - return db, nil -} + selected := &SelectedMailbox{ + Mailbox: mbox, + conn: conn, + readOnly: readOnly, + } -func (u *User) GetMailbox(mbox string) (backend.Mailbox, error) { - fsPath, _, err := u.prepareMboxPath(mbox) + handle, err := u.b.Manager.Mailbox(mbox.mailboxKey(), selected, uids, &recent) if err != nil { - return nil, err + return nil, nil, err } + selected.handle = handle - _, err = os.Stat(fsPath) - if err != nil { - if os.IsNotExist(err) { + return status, selected, nil +} + +func (u *User) Status(mbox string, items []imap.StatusItem) (*imap.MailboxStatus, error) { + mboxObj, ok := u.getMailbox(mbox) + if !ok { + if err := u.validateMboxName(mbox); err != nil { + return nil, err + } + mboxDir, err := u.storage.Dir(mbox) + if err != nil { + return nil, err + } + exists, err := mboxDir.Exists() + if err != nil { + return nil, fmt.Errorf("I/O error: %w", err) + } + if !exists { return nil, backend.ErrNoSuchMailbox } - u.b.Log.Printf("failed to get mailbox: %v", err) - return nil, errors.New("I/O error") + u.mailboxesLock.Lock() + mboxObj = u.ensureMailbox(mbox) + u.mailboxesLock.Unlock() } - handle, err := u.openDB(fsPath, mbox) + entries, err := mboxObj.listEntries() if err != nil { - u.b.Log.Printf("failed to open DB: %v", err) - return nil, errors.New("I/O error, try again more") + return nil, fmt.Errorf("I/O error: %w", err) } - err = handle.Bolt.Update(func(btx *bbolt.Tx) error { - tx := handle.WithTransaction(btx) - var data mboxData - err := tx.One("Dummy", 1, &data) - if err == nil { - return nil + if err := mboxObj.loadMetadataState(); err != nil { + return nil, fmt.Errorf("I/O error: %w", err) + } + + status := imap.NewMailboxStatus(mboxObj.name, items) + baseFlags := []string{imap.SeenFlag, imap.AnsweredFlag, imap.FlaggedFlag, imap.DeletedFlag, imap.DraftFlag} + status.Flags = append([]string{}, baseFlags...) + status.PermanentFlags = append([]string{}, baseFlags...) + status.PermanentFlags = append(status.PermanentFlags, "\\*") + status.UnseenSeqNum = 0 + + flagsMap := map[string]struct{}{} + for _, entry := range entries { + for _, flag := range entry.meta.flags { + flagsMap[flag] = struct{}{} } - if err != storm.ErrNotFound { - return fmt.Errorf("read mboxData: %w", err) + } + flagSeen := map[string]struct{}{} + for _, flag := range status.Flags { + flagSeen[flag] = struct{}{} + } + permSeen := map[string]struct{}{} + for _, flag := range status.PermanentFlags { + permSeen[flag] = struct{}{} + } + for flag := range flagsMap { + if _, ok := flagSeen[flag]; !ok { + status.Flags = append(status.Flags, flag) + flagSeen[flag] = struct{}{} } - u.b.Debug.Printf("initializing %s/%s", u.name, mbox) + if _, ok := permSeen[flag]; !ok { + status.PermanentFlags = append(status.PermanentFlags, flag) + permSeen[flag] = struct{}{} + } + } - data.Dummy = 1 - data.UidNext = 1 - data.UidValidity = uint32(time.Now().UnixNano() & 0xFFFFFFFF) - if err := tx.Save(&data); err != nil { - return fmt.Errorf("save mboxData: %w", err) + for i, entry := range entries { + seqNum := uint32(i + 1) + if !hasFlag(entry.meta.flags, imap.SeenFlag) && status.UnseenSeqNum == 0 { + status.UnseenSeqNum = seqNum } - return nil - }) - if err != nil { - handle.Close() - u.b.Log.Printf("failed to init DB: %v", err) - return nil, errors.New("I/O error, try again later") - } - - u.b.Debug.Printf("get mbox (%v, %v)", mbox, fsPath) - return &Mailbox{ - b: u.b, - name: mbox, - username: u.name, - handle: handle, - dir: maildir.Dir(fsPath), - path: fsPath, - }, nil + } + + for _, item := range items { + switch item { + case imap.StatusMessages: + status.Messages = uint32(len(entries)) + case imap.StatusUidNext: + u.b.statesLock.Lock() + if mboxObj.state.meta != nil { + status.UidNext = mboxObj.state.meta.UIDNext + } else { + status.UidNext = mboxObj.state.uidNext + } + u.b.statesLock.Unlock() + case imap.StatusUidValidity: + status.UidValidity = mboxObj.uidValidity() + case imap.StatusRecent: + for _, entry := range entries { + if entry.meta.recent { + status.Recent++ + } + } + case imap.StatusUnseen: + for _, entry := range entries { + if !hasFlag(entry.meta.flags, imap.SeenFlag) { + status.Unseen++ + } + } + case imap.StatusAppendLimit: + limit := u.mailboxLimit(mboxObj) + if limit == nil { + limit = mboxObj.CreateMessageLimit() + } + if limit != nil { + status.AppendLimit = *limit + } else { + status.AppendLimit = 0 + } + } + } + + return status, nil } -func (u *User) CreateMailbox(mbox string) error { - if strings.EqualFold(mbox, InboxName) { - return backend.ErrMailboxAlreadyExists +func (u *User) SetSubscribed(mbox string, subscribed bool) error { + mboxObj, ok := u.getMailbox(mbox) + if !ok { + return backend.ErrNoSuchMailbox } + mboxObj.subscribed = subscribed + return nil +} - fsPath, _, err := u.prepareMboxPath(mbox) - if err != nil { - return err +func (u *User) CreateMessageLimit() *uint32 { + u.limitLock.Lock() + defer u.limitLock.Unlock() + + if u.appendLimit == nil { + return nil } + val := *u.appendLimit + return &val +} - if _, err := os.Stat(fsPath); err != nil { - if !os.IsNotExist(err) { - u.b.Debug.Printf("failed to create mailbox: %v", err) - return errors.New("I/O error") - } - } else { - return backend.ErrMailboxAlreadyExists +func (u *User) SetMessageLimit(val *uint32) error { + u.limitLock.Lock() + defer u.limitLock.Unlock() + + if val == nil { + u.appendLimit = nil + return nil } + copyVal := *val + u.appendLimit = ©Val + return nil +} - if err := os.MkdirAll(fsPath, 0700); err != nil { - u.b.Debug.Printf("failed to create mailbox: %v", err) - return errors.New("I/O error") +func (u *User) effectiveLimit(selected backend.Mailbox, mbox *Mailbox) *uint32 { + if limit := u.mailboxLimit(mbox); limit != nil { + return limit } - if err := os.MkdirAll(filepath.Join(fsPath, "cur"), 0700); err != nil { - u.b.Debug.Printf("failed to create mailbox: %v", err) - return errors.New("I/O error") + if selected != nil { + if sel, ok := selected.(*SelectedMailbox); ok { + if limit := sel.CreateMessageLimit(); limit != nil { + return limit + } + } } - if err := os.MkdirAll(filepath.Join(fsPath, "new"), 0700); err != nil { - u.b.Debug.Printf("failed to create mailbox: %v", err) - return errors.New("I/O error") + if mbox != nil { + if limit := mbox.CreateMessageLimit(); limit != nil { + return limit + } } - if err := os.MkdirAll(filepath.Join(fsPath, "tmp"), 0700); err != nil { - u.b.Debug.Printf("failed to create mailbox: %v", err) - return errors.New("I/O error") + if limit := u.CreateMessageLimit(); limit != nil { + return limit } - // IMAP index will be created on demand on first SELECT. + return u.b.CreateMessageLimit() +} - u.b.Debug.Printf("create mbox (%v, %v)", mbox, fsPath) +func (u *User) setMailboxLimit(name string, val *uint32) { + u.mboxLimitLock.Lock() + defer u.mboxLimitLock.Unlock() - return nil + if u.mboxLimits == nil { + u.mboxLimits = map[string]*uint32{} + } + if val == nil { + delete(u.mboxLimits, name) + return + } + copyVal := *val + u.mboxLimits[name] = ©Val } -func (u *User) DeleteMailbox(mbox string) error { - if strings.EqualFold(mbox, InboxName) { - return errors.New("cannot delete inbox") +func (u *User) mailboxLimit(mbox *Mailbox) *uint32 { + if mbox == nil { + return nil } + u.mboxLimitLock.Lock() + defer u.mboxLimitLock.Unlock() - fsPath, _, err := u.prepareMboxPath(mbox) - if err != nil { - return err + if u.mboxLimits == nil { + return nil + } + val, ok := u.mboxLimits[mbox.name] + if !ok || val == nil { + return nil + } + copyVal := *val + return ©Val +} + +func (u *User) CreateMessage(mboxName string, flags []string, date time.Time, body imap.Literal, selected backend.Mailbox) error { + mbox, ok := u.getMailbox(mboxName) + if !ok { + return backend.ErrNoSuchMailbox } - if _, err := os.Stat(fsPath); err != nil { - if os.IsNotExist(err) { - return backend.ErrNoSuchMailbox + newFlags := flags[:0] + for _, flag := range flags { + if flag == imap.RecentFlag { + continue } - u.b.Debug.Printf("failed to delete mailbox: %v", err) - return errors.New("I/O error") + newFlags = append(newFlags, flag) } + flags = uniqueFlags(newFlags) - // Delete in that order to - // 1. Prevent IMAP SELECT. - if err := os.RemoveAll(filepath.Join(fsPath, IndexFile)); err != nil { - if !os.IsNotExist(err) { - u.b.Log.Printf("failed to remove mailbox: %v", err) - return errors.New("I/O error") + if date.IsZero() { + date = time.Now() + } + + data, err := io.ReadAll(body) + if err != nil { + return errors.New("I/O error, try again later") + } + + if limit := u.effectiveLimit(selected, mbox); limit != nil { + if uint32(len(data)) > *limit { + return backend.ErrTooBig } } - // 2. Prevent new maildir deliveries. - if err := os.RemoveAll(filepath.Join(fsPath, "tmp")); err != nil { - if !os.IsNotExist(err) { - u.b.Log.Printf("failed to remove mailbox: %v", err) + if err := mbox.loadMetadataState(); err != nil { + return errors.New("I/O error, try again later") + } + + msg, writer, err := mbox.dir.Create(mbox.maildirFlagsFromImap(flags)) + if err != nil { + return errors.New("I/O error, try again later") + } + if _, err := writer.Write(data); err != nil { + _ = writer.Close() + return errors.New("I/O error, try again later") + } + if err := writer.Close(); err != nil { + return errors.New("I/O error, try again later") + } + if err := msg.Chtimes(date, date); err != nil { + u.b.Log.Printf("CreateMessage: chtimes: %v", err) + } + + u.b.statesLock.Lock() + uid := mbox.state.meta.UIDNext + mbox.state.meta.UIDNext++ + mbox.state.meta.UIDByKey[msg.Key()] = uid + mbox.state.meta.FilenameByUID[uid] = msg.Name() + mbox.state.meta.DirtyUIDList = true + meta := &messageMeta{ + uid: uid, + flags: flags, + internalDate: date, + } + mbox.state.uidNext = mbox.state.meta.UIDNext + mbox.state.messages[msg.Key()] = meta + storeRecent := u.b.Manager.NewMessage(mbox.mailboxKey(), meta.uid) + meta.recent = storeRecent + u.b.statesLock.Unlock() + + if err := mbox.writeMetadataState(mbox.state.meta); err != nil { + return errors.New("I/O error, try again later") + } + + return nil +} + +func (u *User) CreateMailbox(name string) error { + if strings.EqualFold(name, InboxName) { + u.mailboxesLock.Lock() + mbox := u.ensureMailbox(InboxName) + u.mailboxesLock.Unlock() + if mbox == nil { return errors.New("I/O error") } + if err := mbox.dir.Init(); err != nil { + return fmt.Errorf("I/O error: %w", err) + } + return nil + } + if err := u.validateMboxName(name); err != nil { + return err + } + mboxDir, err := u.storage.Dir(name) + if err != nil { + return err } - // 3. Prevent in-flight maildir deliveries from completing. - if err := os.RemoveAll(filepath.Join(fsPath, "new")); err != nil { - if !os.IsNotExist(err) { - u.b.Log.Printf("failed to remove mailbox: %v", err) + exists, err := mboxDir.Exists() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + if exists { + return backend.ErrMailboxAlreadyExists + } + + for _, ancestor := range mailboxAncestors(name) { + dir, err := u.storage.Dir(ancestor) + if err != nil { + return err + } + if err := dir.Init(); err != nil { + return fmt.Errorf("I/O error: %w", err) + } + u.mailboxesLock.Lock() + mbox := u.ensureMailbox(ancestor) + u.mailboxesLock.Unlock() + if mbox == nil { return errors.New("I/O error") } } - // ... and remove all messages - if err := os.RemoveAll(filepath.Join(fsPath, "cur")); err != nil { - if !os.IsNotExist(err) { - u.b.Log.Printf("failed to remove mailbox: %v", err) + + if use, ok := defaultMailboxSpecialUse(name); ok { + u.mailboxesLock.Lock() + mbox := u.ensureMailbox(name) + u.mailboxesLock.Unlock() + if mbox == nil { return errors.New("I/O error") } + if err := u.setMailboxSpecialUse(mbox, use); err != nil { + return fmt.Errorf("I/O error: %w", err) + } + } + + return nil +} + +func (u *User) DeleteMailbox(name string) error { + if strings.EqualFold(name, InboxName) { + return errors.New("cannot delete inbox") + } + if err := u.validateMboxName(name); err != nil { + return err + } + mboxDir, err := u.storage.Dir(name) + if err != nil { + return err + } + exists, err := mboxDir.Exists() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + if !exists { + return backend.ErrNoSuchMailbox + } + + childExists := false + children, err := mboxDir.Children() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + if len(children) > 0 { + childExists = true + } + + if err := mboxDir.Remove(childExists); err != nil { + return fmt.Errorf("I/O error: %w", err) } - u.b.Debug.Printf("delete mbox (%v, %v)", mbox, fsPath) + u.mailboxesLock.Lock() + delete(u.mailboxes, name) + u.mailboxesLock.Unlock() + u.b.deleteMailboxState(u.name, name) + u.b.Manager.MailboxDestroyed(u.name + "\x00" + name) return nil } func (u *User) RenameMailbox(existingName, newName string) error { - if strings.EqualFold(existingName, InboxName) { - // TODO: Handle special case of INBOX move. - return errors.New("not implemented") + if err := u.validateMboxName(existingName); err != nil { + return err + } + if err := u.validateMboxName(newName); err != nil { + return err } + if strings.EqualFold(existingName, InboxName) { + if _, ok := u.getMailbox(newName); ok { + return backend.ErrMailboxAlreadyExists + } + destDir, err := u.storage.Dir(newName) + if err != nil { + return err + } + exists, err := destDir.Exists() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + if exists { + return backend.ErrMailboxAlreadyExists + } + if err := destDir.Init(); err != nil { + return fmt.Errorf("I/O error: %w", err) + } + + inbox := u.ensureMailbox(existingName) + dest := u.ensureMailbox(newName) + if inbox == nil || dest == nil { + return errors.New("I/O error") + } - fsPathOld, _, err := u.prepareMboxPath(existingName) + entries, err := inbox.listEntries() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + for _, entry := range entries { + if err := entry.msg.MoveTo(dest.dir); err != nil { + return fmt.Errorf("I/O error: %w", err) + } + } + + u.b.statesLock.Lock() + for key, meta := range inbox.state.messages { + dest.state.messages[key] = meta + } + if dest.state.uidNext < inbox.state.uidNext { + dest.state.uidNext = inbox.state.uidNext + } + inbox.state.messages = map[string]*messageMeta{} + u.b.statesLock.Unlock() + + return nil + } + srcDir, err := u.storage.Dir(existingName) if err != nil { return err } - fsPathNew, _, err := u.prepareMboxPath(newName) + exists, err := srcDir.Exists() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + if !exists { + return backend.ErrNoSuchMailbox + } + destDir, err := u.storage.Dir(newName) if err != nil { return err } + exists, err = destDir.Exists() + if err != nil { + return fmt.Errorf("I/O error: %w", err) + } + if exists { + return backend.ErrMailboxAlreadyExists + } + relName, err := relativeMailboxName(existingName, newName) + if err != nil { + return err + } + if err := srcDir.Rename(relName); err != nil { + return fmt.Errorf("I/O error: %w", err) + } - if err := os.Rename(fsPathOld, fsPathNew); err != nil { - u.b.Log.Printf("failed to rename mailbox: %v", err) - return errors.New("I/O error") + u.mailboxesLock.Lock() + for name, mbox := range u.mailboxes { + if strings.HasPrefix(name, existingName) { + newChild := strings.Replace(name, existingName, newName, 1) + mbox.name = newChild + newDir, err := u.storage.Dir(newChild) + if err != nil { + continue + } + mbox.dir = newDir + u.mailboxes[newChild] = mbox + delete(u.mailboxes, name) + u.b.renameMailboxState(u.name, name, newChild) + u.b.Manager.MailboxDestroyed(u.name + "\x00" + name) + u.b.Manager.MailboxDestroyed(u.name + "\x00" + newChild) + } } - u.b.Debug.Printf("rename mbox (%v, %v), (%v, %v)", existingName, fsPathOld, newName, fsPathNew) + u.mailboxesLock.Unlock() + return nil } func (u *User) Logout() error { - u.b.Debug.Printf("user logged out (%v, %v)", u.name, u.basePath) return nil }