1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

6Layout of storage for accounts:

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

11Index.db holds tables for user information, mailboxes, and messages. Message contents

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "io/fs"

37 "log/slog"

38 "os"

39 "path/filepath"

40 "reflect"

41 "runtime/debug"

42 "slices"

43 "sort"

44 "strconv"

45 "strings"

46 "sync"

47 "time"

49 "golang.org/x/crypto/bcrypt"

50 "golang.org/x/text/secure/precis"

51 "golang.org/x/text/unicode/norm"

53 "github.com/mjl-/bstore"

55 "github.com/mjl-/mox/config"

56 "github.com/mjl-/mox/dns"

57 "github.com/mjl-/mox/junk"

58 "github.com/mjl-/mox/message"

59 "github.com/mjl-/mox/metrics"

60 "github.com/mjl-/mox/mlog"

61 "github.com/mjl-/mox/mox-"

62 "github.com/mjl-/mox/moxio"

63 "github.com/mjl-/mox/moxvar"

64 "github.com/mjl-/mox/publicsuffix"

65 "github.com/mjl-/mox/scram"

66 "github.com/mjl-/mox/smtp"

67)

69// If true, each time an account is closed its database file is checked for

70// consistency. If an inconsistency is found, panic is called. Set by default

71// because of all the packages with tests, the mox main function sets it to

72// false again.

73var CheckConsistencyOnClose = true

75var (

76 ErrUnknownMailbox = errors.New("no such mailbox")

77 ErrUnknownCredentials = errors.New("credentials not found")

78 ErrAccountUnknown = errors.New("no such account")

79 ErrOverQuota = errors.New("account over quota")

80 ErrLoginDisabled = errors.New("login disabled for account")

81)

83var DefaultInitialMailboxes = config.InitialMailboxes{

84 SpecialUse: config.SpecialUseMailboxes{

85 Sent: "Sent",

86 Archive: "Archive",

87 Trash: "Trash",

88 Draft: "Drafts",

89 Junk: "Junk",

90 },

91}

93type SCRAM struct {

94 Salt []byte

95 Iterations int

96 SaltedPassword []byte

97}

99// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

100// block with (a derivation of) the key/password, so we don't store the password in plain

101// text.

102type CRAMMD5 struct {

103 Ipad hash.Hash

104 Opad hash.Hash

105}

106

107// BinaryMarshal is used by bstore to store the ipad/opad hash states.

108func (c CRAMMD5) MarshalBinary() ([]byte, error) {

109 if c.Ipad == nil || c.Opad == nil {

110 return nil, nil

111 }

112

113 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

114 if err != nil {

115 return nil, fmt.Errorf("marshal ipad: %v", err)

116 }

117 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

118 if err != nil {

119 return nil, fmt.Errorf("marshal opad: %v", err)

120 }

121 buf := make([]byte, 2+len(ipad)+len(opad))

122 ipadlen := uint16(len(ipad))

123 buf[0] = byte(ipadlen >> 8)

124 buf[1] = byte(ipadlen >> 0)

125 copy(buf[2:], ipad)

126 copy(buf[2+len(ipad):], opad)

127 return buf, nil

128}

129

130// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

131func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

132 if len(buf) == 0 {

133 *c = CRAMMD5{}

134 return nil

135 }

136 if len(buf) < 2 {

137 return fmt.Errorf("short buffer")

138 }

139 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

140 if len(buf) < 2+ipadlen {

141 return fmt.Errorf("buffer too short for ipadlen")

142 }

143 ipad := md5.New()

144 opad := md5.New()

145 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

146 return fmt.Errorf("unmarshal ipad: %v", err)

147 }

148 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

149 return fmt.Errorf("unmarshal opad: %v", err)

150 }

151 *c = CRAMMD5{ipad, opad}

152 return nil

153}

154

155// Password holds credentials in various forms, for logging in with SMTP/IMAP.

156type Password struct {

157 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

158 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

159 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

160 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

161}

162

163// Subjectpass holds the secret key used to sign subjectpass tokens.

164type Subjectpass struct {

165 Email string // Our destination address (canonical, with catchall localpart stripped).

166 Key string

167}

168

169// NextUIDValidity is a singleton record in the database with the next UIDValidity

170// to use for the next mailbox.

171type NextUIDValidity struct {

172 ID int // Just a single record with ID 1.

173 Next uint32

174}

175

176// SyncState track ModSeqs.

177type SyncState struct {

178 ID int // Just a single record with ID 1.

179

180 // Last used, next assigned will be one higher. The first value we hand out is 2.

181 // That's because 0 (the default value for old existing messages, from before the

182 // Message.ModSeq field) is special in IMAP, so we return it as 1.

183 LastModSeq ModSeq `bstore:"nonzero"`

184

185 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

186 // and requests changes based on a modseq before this one, we don't have the

187 // history to provide information about deletions. We normally keep these expunged

188 // records around, but we may periodically truly delete them to reclaim storage

189 // space. Initially set to -1 because we don't want to match with any ModSeq in the

190 // database, which can be zero values.

191 HighestDeletedModSeq ModSeq

192}

193

194// Mailbox is collection of messages, e.g. Inbox or Sent.

195type Mailbox struct {

196 ID int64

197

198 CreateSeq ModSeq

199 ModSeq ModSeq `bstore:"index"` // Of last change, or when deleted.

200 Expunged bool

201

202 ParentID int64 `bstore:"ref Mailbox"` // Zero for top-level mailbox.

203

204 // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy.

205 // Names must be unique for mailboxes that are not expunged.

206 Name string `bstore:"nonzero"`

207

208 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

209 // name, UIDValidity must be changed. Used by IMAP for synchronization.

210 UIDValidity uint32

211

212 // UID likely to be assigned to next message. Used by IMAP to detect messages

213 // delivered to a mailbox.

214 UIDNext UID

215

216 SpecialUse

217

218 // Keywords as used in messages. Storing a non-system keyword for a message

219 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

220 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

221 // lower case (for JMAP), sorted.

222 Keywords []string

223

224 HaveCounts bool // Deprecated. Covered by Upgrade.MailboxCounts. No longer read.

225 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

226}

227

228// Annotation is a per-mailbox or global (per-account) annotation for the IMAP

229// metadata extension, currently always a private annotation.

230type Annotation struct {

231 ID int64

232

233 CreateSeq ModSeq

234 ModSeq ModSeq `bstore:"index"`

235 Expunged bool

236

237 // Can be zero, indicates global (per-account) annotation.

238 MailboxID int64 `bstore:"ref Mailbox,index MailboxID+Key"`

239

240 // "Entry name", always starts with "/private/" or "/shared/". Stored lower-case,

241 // comparisons must be done case-insensitively.

242 Key string `bstore:"nonzero"`

243

244 IsString bool // If true, the value is a string instead of bytes.

245 Value []byte

246}

247

248// Change returns a broadcastable change for the annotation.

249func (a Annotation) Change(mailboxName string) ChangeAnnotation {

250 return ChangeAnnotation{a.MailboxID, mailboxName, a.Key, a.ModSeq}

251}

252

253// MailboxCounts tracks statistics about messages for a mailbox.

254type MailboxCounts struct {

255 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

256 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

257 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

258 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

259 Size int64 // Number of bytes for all messages.

260}

261

262// MessageCountIMAP returns the total message count for use in IMAP. In IMAP,

263// message marked \Deleted are included, in JMAP they those messages are not

264// visible at all.

265func (mc MailboxCounts) MessageCountIMAP() uint32 {

266 return uint32(mc.Total + mc.Deleted)

267}

268

269func (mc MailboxCounts) String() string {

270 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

271}

272

273// Add increases mailbox counts mc with those of delta.

274func (mc *MailboxCounts) Add(delta MailboxCounts) {

275 mc.Total += delta.Total

276 mc.Deleted += delta.Deleted

277 mc.Unread += delta.Unread

278 mc.Unseen += delta.Unseen

279 mc.Size += delta.Size

280}

281

282// Add decreases mailbox counts mc with those of delta.

283func (mc *MailboxCounts) Sub(delta MailboxCounts) {

284 mc.Total -= delta.Total

285 mc.Deleted -= delta.Deleted

286 mc.Unread -= delta.Unread

287 mc.Unseen -= delta.Unseen

288 mc.Size -= delta.Size

289}

290

291// SpecialUse identifies a specific role for a mailbox, used by clients to

292// understand where messages should go.

293type SpecialUse struct {

294 Archive bool

295 Draft bool // "Drafts"

296 Junk bool

297 Sent bool

298 Trash bool

299}

300

301// UIDNextAdd increases the UIDNext value by n, returning an error on overflow.

302func (mb *Mailbox) UIDNextAdd(n int) error {

303 uidnext := mb.UIDNext + UID(n)

304 if uidnext < mb.UIDNext {

305 return fmt.Errorf("uid overflow on mailbox %q (id %d): uidnext %d, adding %d; consider recreating the mailbox and copying its messages to compact", mb.Name, mb.ID, mb.UIDNext, n)

306 }

307 mb.UIDNext = uidnext

308 return nil

309}

310

311// CalculateCounts calculates the full current counts for messages in the mailbox.

312func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

313 q := bstore.QueryTx[Message](tx)

314 q.FilterNonzero(Message{MailboxID: mb.ID})

315 q.FilterEqual("Expunged", false)

316 err = q.ForEach(func(m Message) error {

317 mc.Add(m.MailboxCounts())

318 return nil

319 })

320 return

321}

322

323// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

324// other connections.

325func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

326 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse, mb.ModSeq}

327}

328

329// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

330// setting a new keyword on a message in the mailbox), for broadcasting to other

331// connections.

332func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

333 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

334}

335

336func (mb Mailbox) ChangeAddMailbox(flags []string) ChangeAddMailbox {

337 return ChangeAddMailbox{Mailbox: mb, Flags: flags}

338}

339

340func (mb Mailbox) ChangeRemoveMailbox() ChangeRemoveMailbox {

341 return ChangeRemoveMailbox{mb.ID, mb.Name, mb.ModSeq}

342}

343

344// KeywordsChanged returns whether the keywords in a mailbox have changed.

345func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

346 if len(mb.Keywords) != len(origmb.Keywords) {

347 return true

348 }

349 // Keywords are stored sorted.

350 for i, kw := range mb.Keywords {

351 if origmb.Keywords[i] != kw {

352 return true

353 }

354 }

355 return false

356}

357

358// CountsChange returns a change with mailbox counts.

359func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

360 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

361}

362

363// Subscriptions are separate from existence of mailboxes.

364type Subscription struct {

365 Name string

366}

367

368// Flags for a mail message.

369type Flags struct {

370 Seen bool

371 Answered bool

372 Flagged bool

373 Forwarded bool

374 Junk bool

375 Notjunk bool

376 Deleted bool

377 Draft bool

378 Phishing bool

379 MDNSent bool

380}

381

382// FlagsAll is all flags set, for use as mask.

383var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

384

385// Validation of "message From" domain.

386type Validation uint8

387

388const (

389 ValidationUnknown Validation = 0

390 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

391 ValidationDMARC Validation = 2 // Actual DMARC policy.

392 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

393 ValidationPass Validation = 4 // For SPF.

394 ValidationNeutral Validation = 5 // For SPF.

395 ValidationTemperror Validation = 6

396 ValidationPermerror Validation = 7

397 ValidationFail Validation = 8

398 ValidationSoftfail Validation = 9 // For SPF.

399 ValidationNone Validation = 10 // E.g. No records.

400)

401

402// Message stored in database and per-message file on disk.

403//

404// Contents are always the combined data from MsgPrefix and the on-disk file named

405// based on ID.

406//

407// Messages always have a header section, even if empty. Incoming messages without

408// header section must get an empty header section added before inserting.

409type Message struct {

410 // ID of the message, determines path to on-disk message file. Set when adding to a

411 // mailbox. When a message is moved to another mailbox, the mailbox ID is changed,

412 // but for synchronization purposes, a new Message record is inserted (which gets a

413 // new ID) with the Expunged field set and the MailboxID and UID copied.

414 ID int64

415

416 // UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per

417 // mailbox. The UID of a message can never change (though messages can be copied),

418 // and the contents of a message/UID also never changes.

419 UID UID `bstore:"nonzero"`

420

421 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

422

423 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

424 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

425 // always <= ModSeq. If Expunged is set, the message has been removed and should not

426 // be returned to the user. In this case, ModSeq is the Seq where the message is

427 // removed, and will never be changed again.

428 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

429 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

430 // The index on CreateSeq helps efficiently finding created messages for JMAP.

431 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

432 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

433 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

434 // return modseq to clients, we turn 0 into 1.

435 ModSeq ModSeq `bstore:"index"`

436 CreateSeq ModSeq `bstore:"index"`

437 Expunged bool

438

439 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

440 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

441 // flag cleared.

442 IsReject bool

443

444 // If set, this is a forwarded message (through a ruleset with IsForward). This

445 // causes fields used during junk analysis to be moved to their Orig variants, and

446 // masked IP fields cleared, so they aren't used in junk classifications for

447 // incoming messages. This ensures the forwarded messages don't cause negative

448 // reputation for the forwarding mail server, which may also be sending regular

449 // messages.

450 IsForward bool

451

452 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

453 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

454 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

455 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

456 // per-mailbox reputation.

457 //

458 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

459 // mailbox, it is set to the intended mailbox according to delivery rules,

460 // typically that of Inbox. When such a message is moved out of Rejects, the

461 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

462 // message is used for reputation calculation for future deliveries to that

463 // mailbox.

464 //

465 // These are not bstore references to prevent having to update all messages in a

466 // mailbox when the original mailbox is removed. Use of these fields requires

467 // checking if the mailbox still exists.

468 MailboxOrigID int64

469 MailboxDestinedID int64

470

471 // Received indicates time of receival over SMTP, or of IMAP APPEND.

472 Received time.Time `bstore:"default now,index"`

473

474 // SaveDate is the time of copy/move/save to a mailbox, used with IMAP SAVEDATE

475 // extension. Must be updated each time a message is copied/moved to another

476 // mailbox. Can be nil for messages from before this functionality was introduced.

477 SaveDate *time.Time `bstore:"default now"`

478

479 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

480 // masked IPs are used to classify incoming messages. They are left empty for

481 // messages matching a ruleset for forwarded messages.

482 RemoteIP string

483 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

484 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

485 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

486

487 // Only set if present and not an IP address. Unicode string. Empty for forwarded

488 // messages.

489 EHLODomain string `bstore:"index EHLODomain+Received"`

490 MailFrom string // With localpart and domain. Can be empty.

491 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

492 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

493 // messages, but see OrigMailFromDomain.

494 MailFromDomain string `bstore:"index MailFromDomain+Received"`

495 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

496 RcptToDomain string // Unicode string.

497

498 // Parsed "From" message header, used for reputation along with domain validation.

499 MsgFromLocalpart smtp.Localpart

500 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

501 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

502

503 // Simplified statements of the Validation fields below, used for incoming messages

504 // to check reputation.

505 EHLOValidated bool

506 MailFromValidated bool

507 MsgFromValidated bool

508

509 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

510 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

511 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

512

513 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

514 // DKIM domain that matched a ruleset's verified domain is left out, but included

515 // in OrigDKIMDomains.

516 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

517

518 // For forwarded messages,

519 OrigEHLODomain string

520 OrigDKIMDomains []string

521

522 // Canonicalized Message-Id, always lower-case and normalized quoting, without

523 // <>'s. Empty if missing. Used for matching message threads, and to prevent

524 // duplicate reject delivery.

525 MessageID string `bstore:"index"`

526 // lower-case: ../rfc/5256:495

527

528 // For matching threads in case there is no References/In-Reply-To header. It is

529 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

530 SubjectBase string `bstore:"index"`

531 // ../rfc/5256:90

532

533 // Hash of message. For rejects delivery in case there is no Message-ID, only set

534 // when delivered as reject.

535 MessageHash []byte

536

537 // ID of message starting this thread.

538 ThreadID int64 `bstore:"index"`

539 // IDs of parent messages, from closest parent to the root message. Parent messages

540 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

541 // never contain the message id itself (a cycle), and parent messages must

542 // reference the same ancestors. Moving a message to another mailbox keeps the

543 // message ID and changes the MailboxID (and UID) of the message, leaving threading

544 // parent ids intact.

545 ThreadParentIDs []int64

546 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

547 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

548 // have been deleted), or subject-based matching was done.

549 ThreadMissingLink bool

550 // If set, newly delivered child messages are automatically marked as read. This

551 // field is copied to new child messages. Changes are propagated to the webmail

552 // client.

553 ThreadMuted bool

554 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

555 // "on" (mode "unread" ignores it). This field is copied to new child message.

556 // Changes are propagated to the webmail client.

557 ThreadCollapsed bool

558

559 // If received message was known to match a mailing list rule (with modified junk

560 // filtering).

561 IsMailingList bool

562

563 // If this message is a DSN, generated by us or received. For DSNs, we don't look

564 // at the subject when matching threads.

565 DSN bool

566

567 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

568 ReceivedTLSCipherSuite uint16

569 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

570

571 Flags

572 // For keywords other than system flags or the basic well-known $-flags. Only in

573 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

574 // (for JMAP), sorted.

575 Keywords []string `bstore:"index"`

576 Size int64

577 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

578 MsgPrefix []byte // Typically holds received headers and/or header separator.

579

580 // If non-nil, a preview of the message based on text and/or html parts of the

581 // message. Used in the webmail and IMAP PREVIEW extension. If non-nil, it is empty

582 // if no preview could be created, or the message has not textual content or

583 // couldn't be parsed.

584 // Previews are typically created when delivering a message, but not when importing

585 // messages, for speed. Previews are generated on first request (in the webmail, or

586 // through the IMAP fetch attribute "PREVIEW" (without "LAZY")), and stored with

587 // the message at that time.

588 // The preview is at most 256 characters (can be more bytes), with detected quoted

589 // text replaced with "[...]". Previews typically end with a newline, callers may

590 // want to strip whitespace.

591 Preview *string

592

593 // ParsedBuf message structure. Currently saved as JSON of message.Part because

594 // bstore wasn't able to store recursive types when this was implemented. Created

595 // when first needed, and saved in the database.

596 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

597 ParsedBuf []byte

598}

599

600// MailboxCounts returns the delta to counts this message means for its

601// mailbox.

602func (m Message) MailboxCounts() (mc MailboxCounts) {

603 if m.Expunged {

604 return

605 }

606 if m.Deleted {

607 mc.Deleted++

608 } else {

609 mc.Total++

610 }

611 if !m.Seen {

612 mc.Unseen++

613 if !m.Deleted {

614 mc.Unread++

615 }

616 }

617 mc.Size += m.Size

618 return

619}

620

621func (m Message) ChangeAddUID(mb Mailbox) ChangeAddUID {

622 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords, mb.MessageCountIMAP(), uint32(mb.MailboxCounts.Unseen)}

623}

624

625func (m Message) ChangeFlags(orig Flags, mb Mailbox) ChangeFlags {

626 mask := m.Flags.Changed(orig)

627 return ChangeFlags{m.MailboxID, m.UID, m.ModSeq, mask, m.Flags, m.Keywords, mb.UIDValidity, uint32(mb.MailboxCounts.Unseen)}

628}

629

630func (m Message) ChangeThread() ChangeThread {

631 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

632}

633

634// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

635// database is sent to the client as 1, because modseq 0 is special in IMAP.

636// ModSeq coming from the client are of type int64.

637type ModSeq int64

638

639func (ms ModSeq) Client() int64 {

640 if ms == 0 {

641 return 1

642 }

643 return int64(ms)

644}

645

646// ModSeqFromClient converts a modseq from a client to a modseq for internal

647// use, e.g. in a database query.

648// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

649func ModSeqFromClient(modseq int64) ModSeq {

650 if modseq == 1 {

651 return 0

652 }

653 return ModSeq(modseq)

654}

655

656// Erase clears fields from a Message that are no longer needed after actually

657// removing the message file from the file system, after all references to the

658// message have gone away. Only the fields necessary for synchronisation are kept.

659func (m *Message) erase() {

660 if !m.Expunged {

661 panic("erase called on non-expunged message")

662 }

663 *m = Message{

664 ID: m.ID,

665 UID: m.UID,

666 MailboxID: m.MailboxID,

667 CreateSeq: m.CreateSeq,

668 ModSeq: m.ModSeq,

669 Expunged: true,

670 ThreadID: m.ThreadID,

671 }

672}

673

674// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based

675// on the part.

676func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

677 m.DSN = part.IsDSN()

678

679 if part.Envelope == nil {

680 return

681 }

682 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

683 if err != nil {

684 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

685 } else if raw {

686 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

687 }

688 m.MessageID = messageID

689 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

690}

691

692// LoadPart returns a message.Part by reading from m.ParsedBuf.

693func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

694 if m.ParsedBuf == nil {

695 return message.Part{}, fmt.Errorf("message not parsed")

696 }

697 var p message.Part

698 err := json.Unmarshal(m.ParsedBuf, &p)

699 if err != nil {

700 return p, fmt.Errorf("unmarshal message part")

701 }

702 p.SetReaderAt(r)

703 return p, nil

704}

705

706// NeedsTraining returns whether message needs a training update, based on

707// TrainedJunk (current training status) and new Junk/Notjunk flags.

708func (m Message) NeedsTraining() bool {

709 needs, _, _, _, _ := m.needsTraining()

710 return needs

711}

712

713func (m Message) needsTraining() (needs, untrain, untrainJunk, train, trainJunk bool) {

714 untrain = m.TrainedJunk != nil

715 untrainJunk = untrain && *m.TrainedJunk

716 train = m.Junk != m.Notjunk

717 trainJunk = m.Junk

718 needs = untrain != train || untrain && train && untrainJunk != trainJunk

719 return

720}

721

722// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

723// used when delivering/moving/copying messages to a mailbox. Mail clients are not

724// very helpful with setting junk/notjunk flags. But clients can move/copy messages

725// to other mailboxes. So we set flags when clients move a message.

726func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

727 if mb.Junk {

728 m.Junk = true

729 m.Notjunk = false

730 return

731 }

732

733 if !conf.AutomaticJunkFlags.Enabled {

734 return

735 }

736

737 lmailbox := strings.ToLower(mb.Name)

738

739 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

740 m.Junk = true

741 m.Notjunk = false

742 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

743 m.Junk = false

744 m.Notjunk = false

745 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

746 m.Junk = false

747 m.Notjunk = true

748 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

749 m.Junk = true

750 m.Notjunk = false

751 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

752 m.Junk = false

753 m.Notjunk = false

754 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

755 m.Junk = false

756 m.Notjunk = true

757 }

758}

759

760// Recipient represents the recipient of a message. It is tracked to allow

761// first-time incoming replies from users this account has sent messages to. When a

762// mailbox is added to the Sent mailbox the message is parsed and recipients are

763// inserted as recipient. Recipients are never removed other than for removing the

764// message. On move/copy of a message, recipients aren't modified either. For IMAP,

765// this assumes a client simply appends messages to the Sent mailbox (as opposed to

766// copying messages from some place).

767type Recipient struct {

768 ID int64

769 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

770 Localpart string `bstore:"nonzero"` // Encoded localpart.

771 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

772 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

773 Sent time.Time `bstore:"nonzero"`

774}

775

776// Outgoing is a message submitted for delivery from the queue. Used to enforce

777// maximum outgoing messages.

778type Outgoing struct {

779 ID int64

780 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

781 Submitted time.Time `bstore:"nonzero,default now"`

782}

783

784// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

785// during most recent connection (delivery attempt).

786type RecipientDomainTLS struct {

787 Domain string // Unicode.

788 Updated time.Time `bstore:"default now"`

789 STARTTLS bool // Supports STARTTLS.

790 RequireTLS bool // Supports RequireTLS SMTP extension.

791}

792

793// DiskUsage tracks quota use.

794type DiskUsage struct {

795 ID int64 // Always one record with ID 1.

796 MessageSize int64 // Sum of all messages, for quota accounting.

797}

798

799// SessionToken and CSRFToken are types to prevent mixing them up.

800// Base64 raw url encoded.

801type SessionToken string

802type CSRFToken string

803

804// LoginSession represents a login session. We keep a limited number of sessions

805// for a user, removing the oldest session when a new one is created.

806type LoginSession struct {

807 ID int64

808 Created time.Time `bstore:"nonzero,default now"` // Of original login.

809 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

810 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

811 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

812 AccountName string `bstore:"nonzero"`

813 LoginAddress string `bstore:"nonzero"`

814

815 // Set when loading from database.

816 sessionToken SessionToken

817 csrfToken CSRFToken

818}

819

820// Quoting is a setting for how to quote in replies/forwards.

821type Quoting string

822

823const (

824 Default Quoting = "" // Bottom-quote if text is selected, top-quote otherwise.

825 Bottom Quoting = "bottom"

826 Top Quoting = "top"

827)

828

829// Settings are webmail client settings.

830type Settings struct {

831 ID uint8 // Singleton ID 1.

832

833 Signature string

834 Quoting Quoting

835

836 // Whether to show the bars underneath the address input fields indicating

837 // starttls/dnssec/dane/mtasts/requiretls support by address.

838 ShowAddressSecurity bool

839

840 // Show HTML version of message by default, instead of plain text.

841 ShowHTML bool

842

843 // If true, don't show shortcuts in webmail after mouse interaction.

844 NoShowShortcuts bool

845

846 // Additional headers to display in message view. E.g. Delivered-To, User-Agent, X-Mox-Reason.

847 ShowHeaders []string

848}

849

850// ViewMode how a message should be viewed: its text parts, html parts, or html

851// with loading external resources.

852type ViewMode string

853

854const (

855 ModeText ViewMode = "text"

856 ModeHTML ViewMode = "html"

857 ModeHTMLExt ViewMode = "htmlext" // HTML with external resources.

858)

859

860// FromAddressSettings are webmail client settings per "From" address.

861type FromAddressSettings struct {

862 FromAddress string // Unicode.

863 ViewMode ViewMode

864}

865

866// RulesetNoListID records a user "no" response to the question of

867// creating/removing a ruleset after moving a message with list-id header from/to

868// the inbox.

869type RulesetNoListID struct {

870 ID int64

871 RcptToAddress string `bstore:"nonzero"`

872 ListID string `bstore:"nonzero"`

873 ToInbox bool // Otherwise from Inbox to other mailbox.

874}

875

876// RulesetNoMsgFrom records a user "no" response to the question of

877// creating/moveing a ruleset after moving a mesage with message "from" address

878// from/to the inbox.

879type RulesetNoMsgFrom struct {

880 ID int64

881 RcptToAddress string `bstore:"nonzero"`

882 MsgFromAddress string `bstore:"nonzero"` // Unicode.

883 ToInbox bool // Otherwise from Inbox to other mailbox.

884}

885

886// RulesetNoMailbox represents a "never from/to this mailbox" response to the

887// question of adding/removing a ruleset after moving a message.

888type RulesetNoMailbox struct {

889 ID int64

890

891 // The mailbox from/to which the move has happened.

892 // Not a references, if mailbox is deleted, an entry becomes ineffective.

893 MailboxID int64 `bstore:"nonzero"`

894 ToMailbox bool // Whether MailboxID is the destination of the move (instead of source).

895}

896

897// MessageErase represents the need to remove a message file from disk, and clear

898// message fields from the database, but only when the last reference to the

899// message is gone (all IMAP sessions need to have applied the changes indicating

900// message removal).

901type MessageErase struct {

902 ID int64 // Same ID as Message.ID.

903

904 // Whether to subtract the size from the total disk usage. Useful for moving

905 // messages, which involves duplicating the message temporarily, while there are

906 // still references in the old mailbox, but which isn't counted as using twice the

907 // disk space..

908 SkipUpdateDiskUsage bool

909}

910

911// Types stored in DB.

912var DBTypes = []any{

913 NextUIDValidity{},

914 Message{},

915 Recipient{},

916 Mailbox{},

917 Subscription{},

918 Outgoing{},

919 Password{},

920 Subjectpass{},

921 SyncState{},

922 Upgrade{},

923 RecipientDomainTLS{},

924 DiskUsage{},

925 LoginSession{},

926 Settings{},

927 FromAddressSettings{},

928 RulesetNoListID{},

929 RulesetNoMsgFrom{},

930 RulesetNoMailbox{},

931 Annotation{},

932 MessageErase{},

933}

934

935// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

936type Account struct {

937 Name string // Name, according to configuration.

938 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

939 DBPath string // Path to database with mailboxes, messages, etc.

940 DB *bstore.DB // Open database connection.

941

942 // Channel that is closed if/when account has/gets "threads" accounting (see

943 // Upgrade.Threads).

944 threadsCompleted chan struct{}

945 // If threads upgrade completed with error, this is set. Used for warning during

946 // delivery, or aborting when importing.

947 threadsErr error

948

949 // Message directory of last delivery. Used to check we don't have to make that

950 // directory when delivering.

951 lastMsgDir string

952

953 // If set, consistency checks won't fail on message ModSeq/CreateSeq being zero.

954 skipMessageZeroSeqCheck bool

955

956 // Write lock must be held when modifying account/mailbox/message/flags/annotations

957 // if the change needs to be synchronized with client connections by broadcasting

958 // the changes. Changes that are not protocol-visible do not require a lock, the

959 // database transactions isolate activity, though locking may be necessary to

960 // protect in-memory-only access.

961 //

962 // Read lock for reading mailboxes/messages as a consistent snapsnot (i.e. not

963 // concurrent changes). For longer transactions, e.g. when reading many messages,

964 // the lock can be released while continuing to read from the transaction.

965 //

966 // When making changes to mailboxes/messages, changes must be broadcasted before

967 // releasing the lock to ensure proper UID ordering.

968 sync.RWMutex

969

970 // Reference count, while >0, this account is alive and shared. Protected by

971 // openAccounts, not by account wlock.

972 nused int

973 removed bool // Marked for removal. Last close removes the account directory.

974 closed chan struct{} // Closed when last reference is gone.

975}

976

977type Upgrade struct {

978 ID byte

979 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

980 MailboxModSeq bool // Whether mailboxes have been assigned modseqs.

981 MailboxParentID bool // Setting ParentID on mailboxes.

982 MailboxCounts bool // Global flag about whether we have mailbox flags. Instead of previous per-mailbox boolean.

983 MessageParseVersion int // If different than latest, all messages will be reparsed.

984}

985

986const MessageParseVersionLatest = 2

987

988// upgradeInit is the value for new account database, which don't need any upgrading.

989var upgradeInit = Upgrade{

990 ID: 1, // Singleton.

991 Threads: 2,

992 MailboxModSeq: true,

993 MailboxParentID: true,

994 MailboxCounts: true,

995 MessageParseVersion: MessageParseVersionLatest,

996}

997

998// InitialUIDValidity returns a UIDValidity used for initializing an account.

999// It can be replaced during tests with a predictable value.

1000var InitialUIDValidity = func() uint32 {

1001 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

1002}

1003

1004var openAccounts = struct {

1005 sync.Mutex

1006 names map[string]*Account

1007}{

1008 names: map[string]*Account{},

1009}

1010

1011func closeAccount(acc *Account) (rerr error) {

1012 // If we need to remove the account files, we do so without the accounts lock.

1013 remove := false

1014 defer func() {

1015 if remove {

1016 log := mlog.New("store", nil)

1017 err := removeAccount(log, acc.Name)

1018 if rerr == nil {

1019 rerr = err

1020 }

1021 close(acc.closed)

1022 }

1023 }()

1024

1025 openAccounts.Lock()

1026 defer openAccounts.Unlock()

1027 acc.nused--

1028 if acc.nused > 0 {

1029 return

1030 }

1031 remove = acc.removed

1032

1033 defer func() {

1034 err := acc.DB.Close()

1035 acc.DB = nil

1036 delete(openAccounts.names, acc.Name)

1037 if !remove {

1038 close(acc.closed)

1039 }

1040

1041 if rerr == nil {

1042 rerr = err

1043 }

1044 }()

1045

1046 // Verify there are no more pending MessageErase records.

1047 l, err := bstore.QueryDB[MessageErase](context.TODO(), acc.DB).List()

1048 if err != nil {

1049 return fmt.Errorf("listing messageerase records: %v", err)

1050 } else if len(l) > 0 {

1051 return fmt.Errorf("messageerase records still present after last account reference is gone: %v", l)

1052 }

1053

1054 return nil

1055}

1056

1057// removeAccount moves the account directory for an account away and removes

1058// all files, and removes the AccountRemove struct from the database.

1059func removeAccount(log mlog.Log, accountName string) error {

1060 log = log.With(slog.String("account", accountName))

1061 log.Info("removing account directory and files")

1062

1063 // First move the account directory away.

1064 odir := filepath.Join(mox.DataDirPath("accounts"), accountName)

1065 tmpdir := filepath.Join(mox.DataDirPath("tmp"), "oldaccount-"+accountName)

1066 if err := os.Rename(odir, tmpdir); err != nil {

1067 return fmt.Errorf("moving account data directory %q out of the way to %q (account not removed): %v", odir, tmpdir, err)

1068 }

1069

1070 var errs []error

1071

1072 // Commit removal to database.

1073 err := AuthDB.Write(context.Background(), func(tx *bstore.Tx) error {

1074 if err := tx.Delete(&AccountRemove{accountName}); err != nil {

1075 return fmt.Errorf("deleting account removal request: %v", err)

1076 }

1077 if err := tlsPublicKeyRemoveForAccount(tx, accountName); err != nil {

1078 return fmt.Errorf("removing tls public keys for account: %v", err)

1079 }

1080

1081 if err := loginAttemptRemoveAccount(tx, accountName); err != nil {

1082 return fmt.Errorf("removing historic login attempts for account: %v", err)

1083 }

1084 return nil

1085 })

1086 if err != nil {

1087 errs = append(errs, fmt.Errorf("remove account from database: %w", err))

1088 }

1089

1090 // Remove the account directory and its message and other files.

1091 if err := os.RemoveAll(tmpdir); err != nil {

1092 errs = append(errs, fmt.Errorf("removing account data directory %q that was moved to %q: %v", odir, tmpdir, err))

1093 }

1094

1095 return errors.Join(errs...)

1096}

1097

1098// OpenAccount opens an account by name.

1099//

1100// No additional data path prefix or ".db" suffix should be added to the name.

1101// A single shared account exists per name.

1102func OpenAccount(log mlog.Log, name string, checkLoginDisabled bool) (*Account, error) {

1103 openAccounts.Lock()

1104 defer openAccounts.Unlock()

1105 if acc, ok := openAccounts.names[name]; ok {

1106 if acc.removed {

1107 return nil, fmt.Errorf("account has been removed")

1108 }

1109

1110 acc.nused++

1111 return acc, nil

1112 }

1113

1114 if a, ok := mox.Conf.Account(name); !ok {

1115 return nil, ErrAccountUnknown

1116 } else if checkLoginDisabled && a.LoginDisabled != "" {

1117 return nil, fmt.Errorf("%w: %s", ErrLoginDisabled, a.LoginDisabled)

1118 }

1119

1120 acc, err := openAccount(log, name)

1121 if err != nil {

1122 return nil, err

1123 }

1124 openAccounts.names[name] = acc

1125 return acc, nil

1126}

1127

1128// openAccount opens an existing account, or creates it if it is missing.

1129// Called with openAccounts lock held.

1130func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

1131 dir := filepath.Join(mox.DataDirPath("accounts"), name)

1132 return OpenAccountDB(log, dir, name)

1133}

1134

1135// OpenAccountDB opens an account database file and returns an initialized account

1136// or error. Only exported for use by subcommands that verify the database file.

1137// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

1138func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

1139 log = log.With(slog.String("account", accountName))

1140

1141 dbpath := filepath.Join(accountDir, "index.db")

1142

1143 // Create account if it doesn't exist yet.

1144 isNew := false

1145 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

1146 isNew = true

1147 os.MkdirAll(accountDir, 0770)

1148 }

1149

1150 opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: moxvar.RegisterLogger(dbpath, log.Logger)}

1151 db, err := bstore.Open(context.TODO(), dbpath, &opts, DBTypes...)

1152 if err != nil {

1153 return nil, err

1154 }

1155

1156 defer func() {

1157 if rerr != nil {

1158 err := db.Close()

1159 log.Check(err, "closing database file after error")

1160 if isNew {

1161 err := os.Remove(dbpath)

1162 log.Check(err, "removing new database file after error")

1163 }

1164 }

1165 }()

1166

1167 acc := &Account{

1168 Name: accountName,

1169 Dir: accountDir,

1170 DBPath: dbpath,

1171 DB: db,

1172 nused: 1,

1173 closed: make(chan struct{}),

1174 threadsCompleted: make(chan struct{}),

1175 }

1176

1177 if isNew {

1178 if err := initAccount(db); err != nil {

1179 return nil, fmt.Errorf("initializing account: %v", err)

1180 }

1181

1182 close(acc.threadsCompleted)

1183 return acc, nil

1184 }

1185

1186 // Ensure singletons are present, like DiskUsage and Settings.

1187 // Process pending MessageErase records. Check that next the message ID assigned by

1188 // the database does not already have a file on disk, or increase the sequence so

1189 // it doesn't.

1190 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

1191 if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent {

1192 if err := tx.Insert(&Settings{ID: 1, ShowAddressSecurity: true}); err != nil {

1193 return err

1194 }

1195 }

1196

1197 du := DiskUsage{ID: 1}

1198 err = tx.Get(&du)

1199 if err == bstore.ErrAbsent {

1200 // No DiskUsage record yet, calculate total size and insert.

1201 err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb Mailbox) error {

1202 du.MessageSize += mb.Size

1203 return nil

1204 })

1205 if err != nil {

1206 return err

1207 }

1208 if err := tx.Insert(&du); err != nil {

1209 return err

1210 }

1211 } else if err != nil {

1212 return err

1213 }

1214

1215 var erase []MessageErase

1216 if _, err := bstore.QueryTx[MessageErase](tx).Gather(&erase).Delete(); err != nil {

1217 return fmt.Errorf("fetching messages to erase: %w", err)

1218 }

1219 if len(erase) > 0 {

1220 log.Debug("deleting message files from message erase records", slog.Int("count", len(erase)))

1221 }

1222 var duChanged bool

1223 for _, me := range erase {

1224 // Clear the fields from the message not needed for synchronization.

1225 m := Message{ID: me.ID}

1226 if err := tx.Get(&m); err != nil {

1227 return fmt.Errorf("get message %d to expunge: %w", me.ID, err)

1228 } else if !m.Expunged {

1229 return fmt.Errorf("message %d to erase is not expunged", m.ID)

1230 }

1231

1232 // We remove before we update/commit the database, so we are sure we don't leave

1233 // files behind in case of an error/crash.

1234 p := acc.MessagePath(me.ID)

1235 err := os.Remove(p)

1236 log.Check(err, "removing message file for expunged message", slog.String("path", p))

1237

1238 if !me.SkipUpdateDiskUsage {

1239 du.MessageSize -= m.Size

1240 duChanged = true

1241 }

1242

1243 m.erase()

1244 if err := tx.Update(&m); err != nil {

1245 return fmt.Errorf("save erase of message %d in database: %w", m.ID, err)

1246 }

1247 }

1248

1249 if duChanged {

1250 if err := tx.Update(&du); err != nil {

1251 return fmt.Errorf("saving disk usage after erasing messages: %w", err)

1252 }

1253 }

1254

1255 // Ensure the message directories don't have a higher message ID than occurs in our

1256 // database. If so, increase the next ID used for inserting a message to prevent

1257 // clash during delivery.

1258 last, err := bstore.QueryTx[Message](tx).SortDesc("ID").Limit(1).Get()

1259 if err != nil && err != bstore.ErrAbsent {

1260 return fmt.Errorf("querying last message: %v", err)

1261 }

1262

1263 // We look in the directory where the message is stored (the id can be 0, which is fine).

1264 maxDBID := last.ID

1265 p := acc.MessagePath(maxDBID)

1266 dir := filepath.Dir(p)

1267 maxFSID := maxDBID

1268 // We also try looking for the next directories that would be created for messages,

1269 // until one doesn't exist anymore. We never delete these directories.

1270 for {

1271 np := acc.MessagePath(maxFSID + msgFilesPerDir)

1272 ndir := filepath.Dir(np)

1273 if _, err := os.Stat(ndir); err == nil {

1274 maxFSID = (maxFSID + msgFilesPerDir) &^ (msgFilesPerDir - 1) // First ID for dir.

1275 dir = ndir

1276 } else if errors.Is(err, fs.ErrNotExist) {

1277 break

1278 } else {

1279 return fmt.Errorf("stat next message directory %q: %v", ndir, err)

1280 }

1281 }

1282 // Find highest numbered file within the directory.

1283 entries, err := os.ReadDir(dir)

1284 if err != nil && !errors.Is(err, fs.ErrNotExist) {

1285 return fmt.Errorf("read message directory %q: %v", dir, err)

1286 }

1287 dirFirstID := maxFSID &^ (msgFilesPerDir - 1)

1288 for _, e := range entries {

1289 id, err := strconv.ParseInt(e.Name(), 10, 64)

1290 if err == nil && (id < dirFirstID || id >= dirFirstID+msgFilesPerDir) {

1291 err = fmt.Errorf("directory %s has message id %d outside of range [%d - %d), ignoring", dir, id, dirFirstID, dirFirstID+msgFilesPerDir)

1292 }

1293 if err != nil {

1294 p := filepath.Join(dir, e.Name())

1295 log.Errorx("unrecognized file in message directory, parsing filename as number", err, slog.String("path", p))

1296 } else {

1297 maxFSID = max(maxFSID, id)

1298 }

1299 }

1300 // Warn if we need to increase the message ID in the database.

1301 var mailboxID int64

1302 if maxFSID > maxDBID {

1303 log.Warn("unexpected message file with higher message id than highest id in database, moving database id sequence forward to prevent clashes during future deliveries", slog.Int64("maxdbmsgid", maxDBID), slog.Int64("maxfilemsgid", maxFSID))

1304

1305 mb, err := bstore.QueryTx[Mailbox](tx).Limit(1).Get()

1306 if err != nil {

1307 return fmt.Errorf("get a mailbox: %v", err)

1308 }

1309 mailboxID = mb.ID

1310 }

1311 for maxFSID > maxDBID {

1312 // Set fields that must be non-zero.

1313 m := Message{

1314 UID: ^UID(0),

1315 MailboxID: mailboxID,

1316 }

1317 // Insert and delete to increase the sequence, silly but effective.

1318 if err := tx.Insert(&m); err != nil {

1319 return fmt.Errorf("inserting message to increase id: %v", err)

1320 }

1321 if err := tx.Delete(&m); err != nil {

1322 return fmt.Errorf("deleting message after increasing id: %v", err)

1323 }

1324 maxDBID = m.ID

1325 }

1326

1327 return nil

1328 })

1329 if err != nil {

1330 return nil, fmt.Errorf("calculating counts for mailbox, inserting settings, expunging messages: %v", err)

1331 }

1332

1333 up := Upgrade{ID: 1}

1334 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

1335 err := tx.Get(&up)

1336 if err == bstore.ErrAbsent {

1337 if err := tx.Insert(&up); err != nil {

1338 return fmt.Errorf("inserting initial upgrade record: %v", err)

1339 }

1340 err = nil

1341 }

1342 return err

1343 })

1344 if err != nil {

1345 return nil, fmt.Errorf("checking message threading: %v", err)

1346 }

1347

1348 // Ensure all mailboxes have a modseq based on highest modseq message in each

1349 // mailbox, and a createseq.

1350 if !up.MailboxModSeq {

1351 log.Debug("upgrade: adding modseq to each mailbox")

1352 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1353 var modseq ModSeq

1354

1355 mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).List()

1356 if err != nil {

1357 return fmt.Errorf("listing mailboxes: %v", err)

1358 }

1359 for _, mb := range mbl {

1360 // Get current highest modseq of message in account.

1361 qms := bstore.QueryTx[Message](tx)

1362 qms.FilterNonzero(Message{MailboxID: mb.ID})

1363 qms.SortDesc("ModSeq")

1364 qms.Limit(1)

1365 m, err := qms.Get()

1366 if err == nil {

1367 mb.ModSeq = ModSeq(m.ModSeq.Client())

1368 } else if err == bstore.ErrAbsent {

1369 if modseq == 0 {

1370 modseq, err = acc.NextModSeq(tx)

1371 if err != nil {

1372 return fmt.Errorf("get next mod seq for mailbox without messages: %v", err)

1373 }

1374 }

1375 mb.ModSeq = modseq

1376 } else {

1377 return fmt.Errorf("looking up highest modseq for mailbox: %v", err)

1378 }

1379 mb.CreateSeq = 1

1380 if err := tx.Update(&mb); err != nil {

1381 return fmt.Errorf("updating mailbox with modseq: %v", err)

1382 }

1383 }

1384

1385 up.MailboxModSeq = true

1386 if err := tx.Update(&up); err != nil {

1387 return fmt.Errorf("marking upgrade done: %v", err)

1388 }

1389

1390 return nil

1391 })

1392 if err != nil {

1393 return nil, fmt.Errorf("upgrade: adding modseq to each mailbox: %v", err)

1394 }

1395 }

1396

1397 // Add ParentID to mailboxes.

1398 if !up.MailboxParentID {

1399 log.Debug("upgrade: setting parentid on each mailbox")

1400

1401 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1402 mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).SortAsc("Name").List()

1403 if err != nil {

1404 return fmt.Errorf("listing mailboxes: %w", err)

1405 }

1406

1407 names := map[string]Mailbox{}

1408 for _, mb := range mbl {

1409 names[mb.Name] = mb

1410 }

1411

1412 var modseq ModSeq

1413

1414 // Ensure a parent mailbox for name exists, creating it if needed, including any

1415 // grandparents, up to the top.

1416 var ensureParentMailboxID func(name string) (int64, error)

1417 ensureParentMailboxID = func(name string) (int64, error) {

1418 parentName := mox.ParentMailboxName(name)

1419 if parentName == "" {

1420 return 0, nil

1421 }

1422 parent := names[parentName]

1423 if parent.ID != 0 {

1424 return parent.ID, nil

1425 }

1426

1427 parentParentID, err := ensureParentMailboxID(parentName)

1428 if err != nil {

1429 return 0, fmt.Errorf("creating parent mailbox %q: %w", parentName, err)

1430 }

1431

1432 if modseq == 0 {

1433 modseq, err = a.NextModSeq(tx)

1434 if err != nil {

1435 return 0, fmt.Errorf("get next modseq: %w", err)

1436 }

1437 }

1438

1439 uidvalidity, err := a.NextUIDValidity(tx)

1440 if err != nil {

1441 return 0, fmt.Errorf("next uid validity: %w", err)

1442 }

1443

1444 parent = Mailbox{

1445 CreateSeq: modseq,

1446 ModSeq: modseq,

1447 ParentID: parentParentID,

1448 Name: parentName,

1449 UIDValidity: uidvalidity,

1450 UIDNext: 1,

1451 SpecialUse: SpecialUse{},

1452 HaveCounts: true,

1453 }

1454 if err := tx.Insert(&parent); err != nil {

1455 return 0, fmt.Errorf("creating parent mailbox: %w", err)

1456 }

1457 return parent.ID, nil

1458 }

1459

1460 for _, mb := range mbl {

1461 parentID, err := ensureParentMailboxID(mb.Name)

1462 if err != nil {

1463 return fmt.Errorf("creating missing parent mailbox for mailbox %q: %w", mb.Name, err)

1464 }

1465 mb.ParentID = parentID

1466 if err := tx.Update(&mb); err != nil {

1467 return fmt.Errorf("update mailbox with parentid: %w", err)

1468 }

1469 }

1470

1471 up.MailboxParentID = true

1472 if err := tx.Update(&up); err != nil {

1473 return fmt.Errorf("marking upgrade done: %w", err)

1474 }

1475 return nil

1476 })

1477 if err != nil {

1478 return nil, fmt.Errorf("upgrade: setting parentid on each mailbox: %w", err)

1479 }

1480 }

1481

1482 if !up.MailboxCounts {

1483 log.Debug("upgrade: ensuring all mailboxes have message counts")

1484

1485 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1486 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

1487 mc, err := mb.CalculateCounts(tx)

1488 if err != nil {

1489 return err

1490 }

1491 mb.HaveCounts = true

1492 mb.MailboxCounts = mc

1493 return tx.Update(&mb)

1494 })

1495 if err != nil {

1496 return err

1497 }

1498

1499 up.MailboxCounts = true

1500 if err := tx.Update(&up); err != nil {

1501 return fmt.Errorf("marking upgrade done: %w", err)

1502 }

1503 return nil

1504 })

1505 if err != nil {

1506 return nil, fmt.Errorf("upgrade: ensuring message counts on all mailboxes")

1507 }

1508 }

1509

1510 if up.MessageParseVersion != MessageParseVersionLatest {

1511 log.Debug("upgrade: reparsing message for mime structures for new message parse version", slog.Int("current", up.MessageParseVersion), slog.Int("latest", MessageParseVersionLatest))

1512

1513 // Unless we also need to upgrade threading, we'll be reparsing messages in the

1514 // background so opening of the account is quick.

1515 done := make(chan error, 1)

1516 bg := up.Threads == 2

1517

1518 // Increase account use before holding on to account in background.

1519 // Caller holds the lock. The goroutine below decreases nused by calling

1520 // closeAccount.

1521 acc.nused++

1522

1523 go func() {

1524 start := time.Now()

1525

1526 var rerr error

1527 defer func() {

1528 x := recover()

1529 if x != nil {

1530 rerr = fmt.Errorf("unhandled panic: %v", x)

1531 log.Error("unhandled panic reparsing messages", slog.Any("err", x))

1532 debug.PrintStack()

1533 metrics.PanicInc(metrics.Store)

1534 }

1535

1536 if bg && rerr != nil {

1537 log.Errorx("upgrade failed: reparsing message for mime structures for new message parse version", rerr, slog.Duration("duration", time.Since(start)))

1538 }

1539 done <- rerr

1540

1541 // Must be done at end of defer. Our parent context/goroutine has openAccounts lock

1542 // held, so we won't make progress until after the enclosing method has returned.

1543 err := closeAccount(acc)

1544 log.Check(err, "closing account after reparsing messages")

1545 }()

1546

1547 var total int

1548 total, rerr = acc.ReparseMessages(mox.Shutdown, log)

1549 if rerr != nil {

1550 rerr = fmt.Errorf("reparsing messages and updating mime structures in message index: %w", rerr)

1551 return

1552 }

1553

1554 up.MessageParseVersion = MessageParseVersionLatest

1555 rerr = acc.DB.Update(context.TODO(), &up)

1556 if rerr != nil {

1557 rerr = fmt.Errorf("marking latest message parse version: %w", rerr)

1558 return

1559 }

1560

1561 log.Info("upgrade completed: reparsing message for mime structures for new message parse version", slog.Int("total", total), slog.Duration("duration", time.Since(start)))

1562 }()

1563

1564 if !bg {

1565 err := <-done

1566 if err != nil {

1567 return nil, err

1568 }

1569 }

1570 }

1571

1572 if up.Threads == 2 {

1573 close(acc.threadsCompleted)

1574 return acc, nil

1575 }

1576

1577 // Increase account use before holding on to account in background.

1578 // Caller holds the lock. The goroutine below decreases nused by calling

1579 // closeAccount.

1580 acc.nused++

1581

1582 // Ensure all messages have a MessageID and SubjectBase, which are needed when

1583 // matching threads.

1584 // Then assign messages to threads, in the same way we do during imports.

1585 log.Info("upgrading account for threading, in background")

1586 go func() {

1587 defer func() {

1588 err := closeAccount(acc)

1589 log.Check(err, "closing use of account after upgrading account storage for threads")

1590

1591 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

1592 close(acc.threadsCompleted)

1593 }()

1594

1595 defer func() {

1596 x := recover() // Should not happen, but don't take program down if it does.

1597 if x != nil {

1598 log.Error("upgradeThreads panic", slog.Any("err", x))

1599 debug.PrintStack()

1600 metrics.PanicInc(metrics.Upgradethreads)

1601 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

1602 }

1603 }()

1604

1605 err := upgradeThreads(mox.Shutdown, log, acc, up)

1606 if err != nil {

1607 a.threadsErr = err

1608 log.Errorx("upgrading account for threading, aborted", err)

1609 } else {

1610 log.Info("upgrading account for threading, completed")

1611 }

1612 }()

1613 return acc, nil

1614}

1615

1616// ThreadingWait blocks until the one-time account threading upgrade for the

1617// account has completed, and returns an error if not successful.

1618//

1619// To be used before starting an import of messages.

1620func (a *Account) ThreadingWait(log mlog.Log) error {

1621 select {

1622 case <-a.threadsCompleted:

1623 return a.threadsErr

1624 default:

1625 }

1626 log.Debug("waiting for account upgrade to complete")

1627

1628 <-a.threadsCompleted

1629 return a.threadsErr

1630}

1631

1632func initAccount(db *bstore.DB) error {

1633 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

1634 uidvalidity := InitialUIDValidity()

1635

1636 if err := tx.Insert(&upgradeInit); err != nil {

1637 return err

1638 }

1639 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

1640 return err

1641 }

1642 if err := tx.Insert(&Settings{ID: 1}); err != nil {

1643 return err

1644 }

1645

1646 modseq, err := nextModSeq(tx)

1647 if err != nil {

1648 return fmt.Errorf("get next modseq: %v", err)

1649 }

1650

1651 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

1652 // Deprecated in favor of InitialMailboxes.

1653 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

1654 mailboxes := []string{"Inbox"}

1655 for _, name := range defaultMailboxes {

1656 if strings.EqualFold(name, "Inbox") {

1657 continue

1658 }

1659 mailboxes = append(mailboxes, name)

1660 }

1661 for _, name := range mailboxes {

1662 mb := Mailbox{

1663 CreateSeq: modseq,

1664 ModSeq: modseq,

1665 ParentID: 0,

1666 Name: name,

1667 UIDValidity: uidvalidity,

1668 UIDNext: 1,

1669 HaveCounts: true,

1670 }

1671 if strings.HasPrefix(name, "Archive") {

1672 mb.Archive = true

1673 } else if strings.HasPrefix(name, "Drafts") {

1674 mb.Draft = true

1675 } else if strings.HasPrefix(name, "Junk") {

1676 mb.Junk = true

1677 } else if strings.HasPrefix(name, "Sent") {

1678 mb.Sent = true

1679 } else if strings.HasPrefix(name, "Trash") {

1680 mb.Trash = true

1681 }

1682 if err := tx.Insert(&mb); err != nil {

1683 return fmt.Errorf("creating mailbox: %w", err)

1684 }

1685 if err := tx.Insert(&Subscription{name}); err != nil {

1686 return fmt.Errorf("adding subscription: %w", err)

1687 }

1688 }

1689 } else {

1690 mailboxes := mox.Conf.Static.InitialMailboxes

1691 var zerouse config.SpecialUseMailboxes

1692 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1693 mailboxes = DefaultInitialMailboxes

1694 }

1695

1696 add := func(name string, use SpecialUse) error {

1697 mb := Mailbox{

1698 CreateSeq: modseq,

1699 ModSeq: modseq,

1700 ParentID: 0,

1701 Name: name,

1702 UIDValidity: uidvalidity,

1703 UIDNext: 1,

1704 SpecialUse: use,

1705 HaveCounts: true,

1706 }

1707 if err := tx.Insert(&mb); err != nil {

1708 return fmt.Errorf("creating mailbox: %w", err)

1709 }

1710 if err := tx.Insert(&Subscription{name}); err != nil {

1711 return fmt.Errorf("adding subscription: %w", err)

1712 }

1713 return nil

1714 }

1715 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1716 if nameOpt == "" {

1717 return nil

1718 }

1719 return add(nameOpt, use)

1720 }

1721 l := []struct {

1722 nameOpt string

1723 use SpecialUse

1724 }{

1725 {"Inbox", SpecialUse{}},

1726 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1727 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1728 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1729 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1730 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1731 }

1732 for _, e := range l {

1733 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1734 return err

1735 }

1736 }

1737 for _, name := range mailboxes.Regular {

1738 if err := add(name, SpecialUse{}); err != nil {

1739 return err

1740 }

1741 }

1742 }

1743

1744 uidvalidity++

1745 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1746 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1747 }

1748 return nil

1749 })

1750}

1751

1752// Remove schedules an account for removal. New opens will fail. When the last

1753// reference is closed, the account files are removed.

1754func (a *Account) Remove(ctx context.Context) error {

1755 openAccounts.Lock()

1756 defer openAccounts.Unlock()

1757

1758 if err := AuthDB.Insert(ctx, &AccountRemove{AccountName: a.Name}); err != nil {

1759 return fmt.Errorf("inserting account removal: %w", err)

1760 }

1761 a.removed = true

1762

1763 return nil

1764}

1765

1766// WaitClosed waits until the last reference to this account is gone and the

1767// account is closed. Used during tests, to ensure the consistency checks run after

1768// expunged messages have been erased.

1769func (a *Account) WaitClosed() {

1770 <-a.closed

1771}

1772

1773// Close reduces the reference count, and closes the database connection when

1774// it was the last user.

1775func (a *Account) Close() error {

1776 if CheckConsistencyOnClose {

1777 xerr := a.CheckConsistency()

1778 err := closeAccount(a)

1779 if xerr != nil {

1780 panic(xerr)

1781 }

1782 return err

1783 }

1784 return closeAccount(a)

1785}

1786

1787// SetSkipMessageModSeqZeroCheck skips consistency checks for Message.ModSeq and

1788// Message.CreateSeq being zero.

1789func (a *Account) SetSkipMessageModSeqZeroCheck(skip bool) {

1790 a.Lock()

1791 defer a.Unlock()

1792 a.skipMessageZeroSeqCheck = true

1793}

1794

1795// CheckConsistency checks the consistency of the database and returns a non-nil

1796// error for these cases:

1797//

1798// - Missing or unexpected on-disk message files.

1799// - Mismatch between message size and length of MsgPrefix and on-disk file.

1800// - Incorrect mailbox counts.

1801// - Incorrect total message size.

1802// - Message with UID >= mailbox uid next.

1803// - Mailbox uidvalidity >= account uid validity.

1804// - Mailbox ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq, and Modseq >= highest message ModSeq.

1805// - Mailbox must have a live parent ID if they are live themselves, live names must be unique.

1806// - Message ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1807// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1808// - Annotations must have ModSeq > 0, CreateSeq > 0, ModSeq >= CreateSeq and live keys must be unique per mailbox.

1809// - Recalculate junk filter (words and counts) and check they are the same.

1810func (a *Account) CheckConsistency() error {

1811 a.Lock()

1812 defer a.Unlock()

1813

1814 var uidErrors []string // With a limit, could be many.

1815 var modseqErrors []string // With limit.

1816 var fileErrors []string // With limit.

1817 var threadidErrors []string // With limit.

1818 var threadParentErrors []string // With limit.

1819 var threadAncestorErrors []string // With limit.

1820 var errmsgs []string

1821

1822 ctx := context.Background()

1823 log := mlog.New("store", nil)

1824

1825 err := a.DB.Read(ctx, func(tx *bstore.Tx) error {

1826 nuv := NextUIDValidity{ID: 1}

1827 err := tx.Get(&nuv)

1828 if err != nil {

1829 return fmt.Errorf("fetching next uid validity: %v", err)

1830 }

1831

1832 mailboxes := map[int64]Mailbox{} // Also expunged mailboxes.

1833 mailboxNames := map[string]Mailbox{} // Only live names.

1834 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1835 mailboxes[mb.ID] = mb

1836 if !mb.Expunged {

1837 if xmb, ok := mailboxNames[mb.Name]; ok {

1838 errmsg := fmt.Sprintf("mailbox %q exists as id %d and id %d", mb.Name, mb.ID, xmb.ID)

1839 errmsgs = append(errmsgs, errmsg)

1840 }

1841 mailboxNames[mb.Name] = mb

1842 }

1843

1844 if mb.UIDValidity >= nuv.Next {

1845 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1846 errmsgs = append(errmsgs, errmsg)

1847 }

1848

1849 if mb.ModSeq == 0 || mb.CreateSeq == 0 || mb.CreateSeq > mb.ModSeq {

1850 errmsg := fmt.Sprintf("mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", mb.Name, mb.ID, mb.ModSeq, mb.CreateSeq)

1851 errmsgs = append(errmsgs, errmsg)

1852 return nil

1853 }

1854 m, err := bstore.QueryTx[Message](tx).FilterNonzero(Message{MailboxID: mb.ID}).SortDesc("ModSeq").Limit(1).Get()

1855 if err == bstore.ErrAbsent {

1856 return nil

1857 } else if err != nil {

1858 return fmt.Errorf("get message with highest modseq for mailbox: %v", err)

1859 } else if mb.ModSeq < m.ModSeq {

1860 errmsg := fmt.Sprintf("mailbox %q (id %d) has modseq %d < highest message modseq is %d", mb.Name, mb.ID, mb.ModSeq, m.ModSeq)

1861 errmsgs = append(errmsgs, errmsg)

1862 }

1863 return nil

1864 })

1865 if err != nil {

1866 return fmt.Errorf("checking mailboxes: %v", err)

1867 }

1868

1869 // Check ParentID and name of parent.

1870 for _, mb := range mailboxNames {

1871 if mox.ParentMailboxName(mb.Name) == "" {

1872 if mb.ParentID == 0 {

1873 continue

1874 }

1875 errmsg := fmt.Sprintf("mailbox %q (id %d) is a root mailbox but has parentid %d", mb.Name, mb.ID, mb.ParentID)

1876 errmsgs = append(errmsgs, errmsg)

1877 } else if mb.ParentID == 0 {

1878 errmsg := fmt.Sprintf("mailbox %q (id %d) is not a root mailbox but has a zero parentid", mb.Name, mb.ID)

1879 errmsgs = append(errmsgs, errmsg)

1880 } else if mox.ParentMailboxName(mb.Name) != mailboxes[mb.ParentID].Name {

1881 errmsg := fmt.Sprintf("mailbox %q (id %d) has parent mailbox id %d with name %q, but parent name should be %q", mb.Name, mb.ID, mb.ParentID, mailboxes[mb.ParentID].Name, mox.ParentMailboxName(mb.Name))

1882 errmsgs = append(errmsgs, errmsg)

1883 }

1884 }

1885

1886 type annotation struct {

1887 mailboxID int64 // Can be 0.

1888 key string

1889 }

1890 annotations := map[annotation]struct{}{}

1891 err = bstore.QueryTx[Annotation](tx).ForEach(func(a Annotation) error {

1892 if !a.Expunged {

1893 k := annotation{a.MailboxID, a.Key}

1894 if _, ok := annotations[k]; ok {

1895 errmsg := fmt.Sprintf("duplicate live annotation key %q for mailbox id %d", a.Key, a.MailboxID)

1896 errmsgs = append(errmsgs, errmsg)

1897 }

1898 annotations[k] = struct{}{}

1899 }

1900 if a.ModSeq == 0 || a.CreateSeq == 0 || a.CreateSeq > a.ModSeq {

1901 errmsg := fmt.Sprintf("annotation %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and modseq >= createseq", a.ID, mailboxes[a.MailboxID].Name, a.MailboxID, a.ModSeq, a.CreateSeq)

1902 errmsgs = append(errmsgs, errmsg)

1903 } else if a.MailboxID > 0 && mailboxes[a.MailboxID].ModSeq < a.ModSeq {

1904 errmsg := fmt.Sprintf("annotation %d in mailbox %q (id %d) has invalid modseq %d > mailbox modseq %d", a.ID, mailboxes[a.MailboxID].Name, a.MailboxID, a.ModSeq, mailboxes[a.MailboxID].ModSeq)

1905 errmsgs = append(errmsgs, errmsg)

1906 }

1907 return nil

1908 })

1909 if err != nil {

1910 return fmt.Errorf("checking mailbox annotations: %v", err)

1911 }

1912

1913 // All message id's from database. For checking for unexpected files afterwards.

1914 messageIDs := map[int64]struct{}{}

1915 eraseMessageIDs := map[int64]bool{} // Value indicates whether to skip updating disk usage.

1916

1917 // If configured, we'll be building up the junk filter for the messages, to compare

1918 // against the on-disk junk filter.

1919 var jf *junk.Filter

1920 conf, _ := a.Conf()

1921 if conf.JunkFilter != nil {

1922 random := make([]byte, 16)

1923 cryptorand.Read(random)

1924 dbpath := filepath.Join(mox.DataDirPath("tmp"), fmt.Sprintf("junkfilter-check-%x.db", random))

1925 bloompath := filepath.Join(mox.DataDirPath("tmp"), fmt.Sprintf("junkfilter-check-%x.bloom", random))

1926 os.MkdirAll(filepath.Dir(dbpath), 0700)

1927 defer func() {

1928 err := os.Remove(bloompath)

1929 log.Check(err, "removing temp bloom file")

1930 err = os.Remove(dbpath)

1931 log.Check(err, "removing temp junk filter database file")

1932 }()

1933 jf, err = junk.NewFilter(ctx, log, conf.JunkFilter.Params, dbpath, bloompath)

1934 if err != nil {

1935 return fmt.Errorf("new junk filter: %v", err)

1936 }

1937 defer func() {

1938 err := jf.Close()

1939 log.Check(err, "closing junk filter")

1940 }()

1941 }

1942 var ntrained int

1943

1944 // Get IDs of erase messages not yet removed, they'll have a message file.

1945 err = bstore.QueryTx[MessageErase](tx).ForEach(func(me MessageErase) error {

1946 eraseMessageIDs[me.ID] = me.SkipUpdateDiskUsage

1947 return nil

1948 })

1949 if err != nil {

1950 return fmt.Errorf("listing message erase records")

1951 }

1952

1953 counts := map[int64]MailboxCounts{}

1954 var totalExpungedSize int64

1955 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1956 mc := counts[m.MailboxID]

1957 mc.Add(m.MailboxCounts())

1958 counts[m.MailboxID] = mc

1959

1960 mb := mailboxes[m.MailboxID]

1961

1962 if (!a.skipMessageZeroSeqCheck && (m.ModSeq == 0 || m.CreateSeq == 0) || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1963 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1964 modseqErrors = append(modseqErrors, modseqerr)

1965 }

1966 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1967 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1968 uidErrors = append(uidErrors, uiderr)

1969 }

1970 if m.Expunged {

1971 if skip := eraseMessageIDs[m.ID]; !skip {

1972 totalExpungedSize += m.Size

1973 }

1974 return nil

1975 }

1976

1977 messageIDs[m.ID] = struct{}{}

1978 p := a.MessagePath(m.ID)

1979 st, err := os.Stat(p)

1980 if err != nil {

1981 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1982 fileErrors = append(fileErrors, existserr)

1983 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1984 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1985 fileErrors = append(fileErrors, sizeerr)

1986 }

1987

1988 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1989 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1990 threadidErrors = append(threadidErrors, err)

1991 }

1992 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1993 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1994 threadParentErrors = append(threadParentErrors, err)

1995 }

1996 for i, pid := range m.ThreadParentIDs {

1997 am := Message{ID: pid}

1998 if err := tx.Get(&am); err == bstore.ErrAbsent || err == nil && am.Expunged {

1999 continue

2000 } else if err != nil {

2001 return fmt.Errorf("get ancestor message: %v", err)

2002 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

2003 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

2004 threadAncestorErrors = append(threadAncestorErrors, err)

2005 } else {

2006 break

2007 }

2008 }

2009

2010 if jf != nil {

2011 if m.Junk != m.Notjunk {

2012 ntrained++

2013 if _, err := a.TrainMessage(ctx, log, jf, m.Notjunk, m); err != nil {

2014 return fmt.Errorf("train message: %v", err)

2015 }

2016 // We are not setting m.TrainedJunk, we were only recalculating the words.

2017 }

2018 }

2019

2020 return nil

2021 })

2022 if err != nil {

2023 return fmt.Errorf("reading messages: %v", err)

2024 }

2025

2026 msgdir := filepath.Join(a.Dir, "msg")

2027 err = filepath.WalkDir(msgdir, func(path string, entry fs.DirEntry, err error) error {

2028 if err != nil {

2029 if path == msgdir && errors.Is(err, fs.ErrNotExist) {

2030 return nil

2031 }

2032 return err

2033 }

2034 if entry.IsDir() {

2035 return nil

2036 }

2037 id, err := strconv.ParseInt(filepath.Base(path), 10, 64)

2038 if err != nil {

2039 return fmt.Errorf("parsing message id from path %q: %v", path, err)

2040 }

2041 _, mok := messageIDs[id]

2042 _, meok := eraseMessageIDs[id]

2043 if !mok && !meok {

2044 return fmt.Errorf("unexpected message file %q", path)

2045 }

2046 return nil

2047 })

2048 if err != nil {

2049 return fmt.Errorf("walking message dir: %v", err)

2050 }

2051

2052 var totalMailboxSize int64

2053 for _, mb := range mailboxNames {

2054 totalMailboxSize += mb.Size

2055 if mb.MailboxCounts != counts[mb.ID] {

2056 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

2057 errmsgs = append(errmsgs, mbcounterr)

2058 }

2059 }

2060

2061 du := DiskUsage{ID: 1}

2062 if err := tx.Get(&du); err != nil {

2063 return fmt.Errorf("get diskusage")

2064 }

2065 if du.MessageSize != totalMailboxSize+totalExpungedSize {

2066 errmsg := fmt.Sprintf("total disk usage message size in database is %d != sum of mailbox message sizes %d + sum unerased expunged message sizes %d", du.MessageSize, totalMailboxSize, totalExpungedSize)

2067 errmsgs = append(errmsgs, errmsg)

2068 }

2069

2070 // Compare on-disk junk filter with our recalculated filter.

2071 if jf != nil {

2072 load := func(f *junk.Filter) (map[junk.Wordscore]struct{}, error) {

2073 words := map[junk.Wordscore]struct{}{}

2074 err := bstore.QueryDB[junk.Wordscore](ctx, f.DB()).ForEach(func(w junk.Wordscore) error {

2075 if w.Ham != 0 || w.Spam != 0 {

2076 words[w] = struct{}{}

2077 }

2078 return nil

2079 })

2080 if err != nil {

2081 return nil, fmt.Errorf("read junk filter wordscores: %v", err)

2082 }

2083 return words, nil

2084 }

2085 if err := jf.Save(); err != nil {

2086 return fmt.Errorf("save recalculated junk filter: %v", err)

2087 }

2088 wordsExp, err := load(jf)

2089 if err != nil {

2090 return fmt.Errorf("read recalculated junk filter: %v", err)

2091 }

2092

2093 ajf, _, err := a.OpenJunkFilter(ctx, log)

2094 if err != nil {

2095 return fmt.Errorf("open account junk filter: %v", err)

2096 }

2097 defer func() {

2098 err := ajf.Close()

2099 log.Check(err, "closing junk filter")

2100 }()

2101 wordsGot, err := load(ajf)

2102 if err != nil {

2103 return fmt.Errorf("read account junk filter: %v", err)

2104 }

2105

2106 if !reflect.DeepEqual(wordsGot, wordsExp) {

2107 errmsg := fmt.Sprintf("unexpected values in junk filter, trained %d of %d\ngot:\n%v\nexpected:\n%v", ntrained, len(messageIDs), wordsGot, wordsExp)

2108 errmsgs = append(errmsgs, errmsg)

2109 }

2110 }

2111

2112 return nil

2113 })

2114 if err != nil {

2115 return err

2116 }

2117 errmsgs = append(errmsgs, uidErrors...)

2118 errmsgs = append(errmsgs, modseqErrors...)

2119 errmsgs = append(errmsgs, fileErrors...)

2120 errmsgs = append(errmsgs, threadidErrors...)

2121 errmsgs = append(errmsgs, threadParentErrors...)

2122 errmsgs = append(errmsgs, threadAncestorErrors...)

2123 if len(errmsgs) > 0 {

2124 return fmt.Errorf("%s", strings.Join(errmsgs, "; "))

2125 }

2126 return nil

2127}

2128

2129// Conf returns the configuration for this account if it still exists. During

2130// an SMTP session, a configuration update may drop an account.

2131func (a *Account) Conf() (config.Account, bool) {

2132 return mox.Conf.Account(a.Name)

2133}

2134

2135// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

2136func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

2137 nuv := NextUIDValidity{ID: 1}

2138 if err := tx.Get(&nuv); err != nil {

2139 return 0, err

2140 }

2141 v := nuv.Next

2142 nuv.Next++

2143 if err := tx.Update(&nuv); err != nil {

2144 return 0, err

2145 }

2146 return v, nil

2147}

2148

2149// NextModSeq returns the next modification sequence, which is global per account,

2150// over all types.

2151func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

2152 return nextModSeq(tx)

2153}

2154

2155func nextModSeq(tx *bstore.Tx) (ModSeq, error) {

2156 v := SyncState{ID: 1}

2157 if err := tx.Get(&v); err == bstore.ErrAbsent {

2158 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

2159 // already used.

2160 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

2161 // makes sense.

2162 v = SyncState{1, 2, -1}

2163 return v.LastModSeq, tx.Insert(&v)

2164 } else if err != nil {

2165 return 0, err

2166 }

2167 v.LastModSeq++

2168 return v.LastModSeq, tx.Update(&v)

2169}

2170

2171func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

2172 v := SyncState{ID: 1}

2173 err := tx.Get(&v)

2174 if err == bstore.ErrAbsent {

2175 return 0, nil

2176 }

2177 return v.HighestDeletedModSeq, err

2178}

2179

2180// WithWLock runs fn with account writelock held. Necessary for account/mailbox

2181// modification. For message delivery, a read lock is required.

2182func (a *Account) WithWLock(fn func()) {

2183 a.Lock()

2184 defer a.Unlock()

2185 fn()

2186}

2187

2188// WithRLock runs fn with account read lock held. Needed for message delivery.

2189func (a *Account) WithRLock(fn func()) {

2190 a.RLock()

2191 defer a.RUnlock()

2192 fn()

2193}

2194

2195// AddOpts influence which work MessageAdd does. Some callers can batch

2196// checks/operations efficiently. For convenience and safety, a zero AddOpts does

2197// all the checks and work.

2198type AddOpts struct {

2199 SkipCheckQuota bool

2200

2201 // If set, the message size is not added to the disk usage. Caller must do that,

2202 // e.g. for many messages at once. If used together with SkipCheckQuota, the

2203 // DiskUsage is not read for database when adding a message.

2204 SkipUpdateDiskUsage bool

2205

2206 // Do not fsync the delivered message file. Useful when copying message files from

2207 // another mailbox. The hardlink created during delivery only needs a directory

2208 // fsync.

2209 SkipSourceFileSync bool

2210

2211 // The directory in which the message file is delivered, typically with a hard

2212 // link, is not fsynced. Useful when delivering many files. A single or few

2213 // directory fsyncs are more efficient.

2214 SkipDirSync bool

2215

2216 // Do not assign thread information to a message. Useful when importing many

2217 // messages and assigning threads efficiently after importing messages.

2218 SkipThreads bool

2219

2220 // If JunkFilter is set, it is used for training. If not set, and the filter must

2221 // be trained for a message, the junk filter is opened, modified and saved to disk.

2222 JunkFilter *junk.Filter

2223

2224 SkipTraining bool

2225

2226 // If true, a preview will be generated if the Message doesn't already have one.

2227 SkipPreview bool

2228}

2229

2230// todo optimization: when moving files, we open the original, call MessageAdd() which hardlinks it and close the file gain. when passing the filename, we could just use os.Link, saves 2 syscalls.

2231

2232// MessageAdd delivers a mail message to the account.

2233//

2234// The file is hardlinked or copied, the caller must clean up the original file. If

2235// this call succeeds, but the database transaction with the change can't be

2236// committed, the caller must clean up the delivered message file identified by

2237// m.ID.

2238//

2239// If the message does not fit in the quota, an error with ErrOverQuota is returned

2240// and the mailbox and message are unchanged and the transaction can continue. For

2241// other errors, the caller must abort the transaction.

2242//

2243// The message, with msg.MsgPrefix and msgFile combined, must have a header

2244// section. The caller is responsible for adding a header separator to

2245// msg.MsgPrefix if missing from an incoming message.

2246//

2247// If UID is not set, it is assigned automatically.

2248//

2249// If the message ModSeq is zero, it is assigned automatically. If the message

2250// CreateSeq is zero, it is set to ModSeq. The mailbox ModSeq is set to the message

2251// ModSeq.

2252//

2253// If the message does not fit in the quota, an error with ErrOverQuota is returned

2254// and the mailbox and message are unchanged and the transaction can continue. For

2255// other errors, the caller must abort the transaction.

2256//

2257// If the destination mailbox has the Sent special-use flag, the message is parsed

2258// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

2259// reputation classification.

2260//

2261// Must be called with account write lock held.

2262//

2263// Caller must save the mailbox after MessageAdd returns, and broadcast changes for

2264// new the message, updated mailbox counts and possibly new mailbox keywords.

2265func (a *Account) MessageAdd(log mlog.Log, tx *bstore.Tx, mb *Mailbox, m *Message, msgFile *os.File, opts AddOpts) (rerr error) {

2266 if m.Expunged {

2267 return fmt.Errorf("cannot deliver expunged message")

2268 }

2269

2270 if !opts.SkipUpdateDiskUsage || !opts.SkipCheckQuota {

2271 du := DiskUsage{ID: 1}

2272 if err := tx.Get(&du); err != nil {

2273 return fmt.Errorf("get disk usage: %v", err)

2274 }

2275

2276 if !opts.SkipCheckQuota {

2277 maxSize := a.QuotaMessageSize()

2278 if maxSize > 0 && m.Size > maxSize-du.MessageSize {

2279 return fmt.Errorf("%w: max size %d bytes", ErrOverQuota, maxSize)

2280 }

2281 }

2282

2283 if !opts.SkipUpdateDiskUsage {

2284 du.MessageSize += m.Size

2285 if err := tx.Update(&du); err != nil {

2286 return fmt.Errorf("update disk usage: %v", err)

2287 }

2288 }

2289 }

2290

2291 m.MailboxID = mb.ID

2292 if m.MailboxOrigID == 0 {

2293 m.MailboxOrigID = mb.ID

2294 }

2295 if m.UID == 0 {

2296 m.UID = mb.UIDNext

2297 if err := mb.UIDNextAdd(1); err != nil {

2298 return fmt.Errorf("adding uid: %v", err)

2299 }

2300 }

2301 if m.ModSeq == 0 {

2302 modseq, err := a.NextModSeq(tx)

2303 if err != nil {

2304 return fmt.Errorf("assigning next modseq: %w", err)

2305 }

2306 m.ModSeq = modseq

2307 } else if m.ModSeq < mb.ModSeq {

2308 return fmt.Errorf("cannot deliver message with modseq %d < mailbox modseq %d", m.ModSeq, mb.ModSeq)

2309 }

2310 if m.CreateSeq == 0 {

2311 m.CreateSeq = m.ModSeq

2312 }

2313 mb.ModSeq = m.ModSeq

2314

2315 if m.SaveDate == nil {

2316 now := time.Now()

2317 m.SaveDate = &now

2318 }

2319 if m.Received.IsZero() {

2320 m.Received = time.Now()

2321 }

2322

2323 if len(m.Keywords) > 0 {

2324 mb.Keywords, _ = MergeKeywords(mb.Keywords, m.Keywords)

2325 }

2326

2327 conf, _ := a.Conf()

2328 m.JunkFlagsForMailbox(*mb, conf)

2329

2330 var part *message.Part

2331 if m.ParsedBuf == nil {

2332 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

2333 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

2334 if err != nil {

2335 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

2336 // We continue, p is still valid.

2337 }

2338 part = &p

2339 buf, err := json.Marshal(part)

2340 if err != nil {

2341 return fmt.Errorf("marshal parsed message: %w", err)

2342 }

2343 m.ParsedBuf = buf

2344 }

2345

2346 var partTried bool

2347 getPart := func() *message.Part {

2348 if part != nil {

2349 return part

2350 }

2351 if partTried {

2352 return nil

2353 }

2354 partTried = true

2355 var p message.Part

2356 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

2357 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

2358 } else {

2359 mr := FileMsgReader(m.MsgPrefix, msgFile)

2360 p.SetReaderAt(mr)

2361 part = &p

2362 }

2363 return part

2364 }

2365

2366 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

2367 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

2368 m.MailboxDestinedID = 0

2369 }

2370

2371 if m.MessageID == "" && m.SubjectBase == "" && getPart() != nil {

2372 m.PrepareThreading(log, part)

2373 }

2374

2375 if !opts.SkipPreview && m.Preview == nil {

2376 if p := getPart(); p != nil {

2377 s, err := p.Preview(log)

2378 if err != nil {

2379 return fmt.Errorf("generating preview: %v", err)

2380 }

2381 m.Preview = &s

2382 }

2383 }

2384

2385 // Assign to thread (if upgrade has completed).

2386 noThreadID := opts.SkipThreads

2387 if m.ThreadID == 0 && !opts.SkipThreads && getPart() != nil {

2388 select {

2389 case <-a.threadsCompleted:

2390 if a.threadsErr != nil {

2391 log.Info("not assigning threads for new delivery, upgrading to threads failed")

2392 noThreadID = true

2393 } else {

2394 if err := assignThread(log, tx, m, part); err != nil {

2395 return fmt.Errorf("assigning thread: %w", err)

2396 }

2397 }

2398 default:

2399 // note: since we have a write transaction to get here, we can't wait for the

2400 // thread upgrade to finish.

2401 // If we don't assign a threadid the upgrade process will do it.

2402 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

2403 noThreadID = true

2404 }

2405 }

2406

2407 if err := tx.Insert(m); err != nil {

2408 return fmt.Errorf("inserting message: %w", err)

2409 }

2410 if !noThreadID && m.ThreadID == 0 {

2411 m.ThreadID = m.ID

2412 if err := tx.Update(m); err != nil {

2413 return fmt.Errorf("updating message for its own thread id: %w", err)

2414 }

2415 }

2416

2417 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's if the mail client doesn't save a message that includes the bcc header in the sent mailbox.

2418 if mb.Sent && getPart() != nil && part.Envelope != nil {

2419 e := part.Envelope

2420 sent := e.Date

2421 if sent.IsZero() {

2422 sent = m.Received

2423 }

2424 if sent.IsZero() {

2425 sent = time.Now()

2426 }

2427 addrs := append(append(e.To, e.CC...), e.BCC...)

2428 for _, addr := range addrs {

2429 if addr.User == "" {

2430 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

2431 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

2432 continue

2433 }

2434 d, err := dns.ParseDomain(addr.Host)

2435 if err != nil {

2436 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

2437 continue

2438 }

2439 lp, err := smtp.ParseLocalpart(addr.User)

2440 if err != nil {

2441 log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))

2442 continue

2443 }

2444 mr := Recipient{

2445 MessageID: m.ID,

2446 Localpart: lp.String(),

2447 Domain: d.Name(),

2448 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

2449 Sent: sent,

2450 }

2451 if err := tx.Insert(&mr); err != nil {

2452 return fmt.Errorf("inserting sent message recipients: %w", err)

2453 }

2454 }

2455 }

2456

2457 msgPath := a.MessagePath(m.ID)

2458 msgDir := filepath.Dir(msgPath)

2459 if a.lastMsgDir != msgDir {

2460 os.MkdirAll(msgDir, 0770)

2461 if err := moxio.SyncDir(log, msgDir); err != nil {

2462 return fmt.Errorf("sync message dir: %w", err)

2463 }

2464 a.lastMsgDir = msgDir

2465 }

2466

2467 // Sync file data to disk.

2468 if !opts.SkipSourceFileSync {

2469 if err := msgFile.Sync(); err != nil {

2470 return fmt.Errorf("fsync message file: %w", err)

2471 }

2472 }

2473

2474 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

2475 return fmt.Errorf("linking/copying message to new file: %w", err)

2476 }

2477

2478 defer func() {

2479 if rerr != nil {

2480 err := os.Remove(msgPath)

2481 log.Check(err, "removing delivered message file", slog.String("path", msgPath))

2482 }

2483 }()

2484

2485 if !opts.SkipDirSync {

2486 if err := moxio.SyncDir(log, msgDir); err != nil {

2487 return fmt.Errorf("sync directory: %w", err)

2488 }

2489 }

2490

2491 if !opts.SkipTraining && m.NeedsTraining() && a.HasJunkFilter() {

2492 jf, opened, err := a.ensureJunkFilter(context.TODO(), log, opts.JunkFilter)

2493 if err != nil {

2494 return fmt.Errorf("open junk filter: %w", err)

2495 }

2496 defer func() {

2497 if jf != nil && opened {

2498 err := jf.CloseDiscard()

2499 log.Check(err, "closing junk filter without saving")

2500 }

2501 }()

2502

2503 // todo optimize: should let us do the tx.Update of m if needed. we should at least merge it with the common case of setting a thread id. and we should try to merge that with the insert by expliciting getting the next id from bstore.

2504

2505 if err := a.RetrainMessage(context.TODO(), log, tx, jf, m); err != nil {

2506 return fmt.Errorf("training junkfilter: %w", err)

2507 }

2508

2509 if opened {

2510 err := jf.Close()

2511 jf = nil

2512 if err != nil {

2513 return fmt.Errorf("close junk filter: %w", err)

2514 }

2515 }

2516 }

2517

2518 mb.MailboxCounts.Add(m.MailboxCounts())

2519

2520 return nil

2521}

2522

2523// SetPassword saves a new password for this account. This password is used for

2524// IMAP, SMTP (submission) sessions and the HTTP account web page.

2525//

2526// Callers are responsible for checking if the account has NoCustomPassword set.

2527func (a *Account) SetPassword(log mlog.Log, password string) error {

2528 password, err := precis.OpaqueString.String(password)

2529 if err != nil {

2530 return fmt.Errorf(`password not allowed by "precis"`)

2531 }

2532

2533 if len(password) < 8 {

2534 // We actually check for bytes...

2535 return fmt.Errorf("password must be at least 8 characters long")

2536 }

2537

2538 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

2539 if err != nil {

2540 return fmt.Errorf("generating password hash: %w", err)

2541 }

2542

2543 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2544 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

2545 return fmt.Errorf("deleting existing password: %v", err)

2546 }

2547 var pw Password

2548 pw.Hash = string(hash)

2549

2550 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

2551 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

2552 // first block is based on the key/password. We hash those first blocks now, and

2553 // store the hash state in the database. When we actually authenticate, we'll

2554 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

2555 // because it does not expose its internal state and isn't a BinaryMarshaler.

2556 // ../rfc/2104:121

2557 pw.CRAMMD5.Ipad = md5.New()

2558 pw.CRAMMD5.Opad = md5.New()

2559 key := []byte(password)

2560 if len(key) > 64 {

2561 t := md5.Sum(key)

2562 key = t[:]

2563 }

2564 ipad := make([]byte, md5.BlockSize)

2565 opad := make([]byte, md5.BlockSize)

2566 copy(ipad, key)

2567 copy(opad, key)

2568 for i := range ipad {

2569 ipad[i] ^= 0x36

2570 opad[i] ^= 0x5c

2571 }

2572 pw.CRAMMD5.Ipad.Write(ipad)

2573 pw.CRAMMD5.Opad.Write(opad)

2574

2575 pw.SCRAMSHA1.Salt = scram.MakeRandom()

2576 pw.SCRAMSHA1.Iterations = 2 * 4096

2577 pw.SCRAMSHA1.SaltedPassword, err = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

2578 if err != nil {

2579 return fmt.Errorf("scram sha1 salt password: %w", err)

2580 }

2581

2582 pw.SCRAMSHA256.Salt = scram.MakeRandom()

2583 pw.SCRAMSHA256.Iterations = 4096

2584 pw.SCRAMSHA256.SaltedPassword, err = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

2585 if err != nil {

2586 return fmt.Errorf("scram sha256 salt password: %w", err)

2587 }

2588

2589 if err := tx.Insert(&pw); err != nil {

2590 return fmt.Errorf("inserting new password: %v", err)

2591 }

2592

2593 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

2594 })

2595 if err == nil {

2596 log.Info("new password set for account", slog.String("account", a.Name))

2597 }

2598 return err

2599}

2600

2601// SessionsClear invalidates all (web) login sessions for the account.

2602func (a *Account) SessionsClear(ctx context.Context, log mlog.Log) error {

2603 return a.DB.Write(ctx, func(tx *bstore.Tx) error {

2604 return sessionRemoveAll(ctx, log, tx, a.Name)

2605 })

2606}

2607

2608// Subjectpass returns the signing key for use with subjectpass for the given

2609// email address with canonical localpart.

2610func (a *Account) Subjectpass(email string) (key string, err error) {

2611 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2612 v := Subjectpass{Email: email}

2613 err := tx.Get(&v)

2614 if err == nil {

2615 key = v.Key

2616 return nil

2617 }

2618 if !errors.Is(err, bstore.ErrAbsent) {

2619 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

2620 }

2621 key = ""

2622 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

2623 buf := make([]byte, 16)

2624 cryptorand.Read(buf)

2625 for _, b := range buf {

2626 key += string(chars[int(b)%len(chars)])

2627 }

2628 v.Key = key

2629 return tx.Insert(&v)

2630 })

2631}

2632

2633// Ensure mailbox is present in database, adding records for the mailbox and its

2634// parents if they aren't present.

2635//

2636// If subscribe is true, any mailboxes that were created will also be subscribed to.

2637//

2638// The leaf mailbox is created with special-use flags, taking the flags away from

2639// other mailboxes, and reflecting that in the returned changes.

2640//

2641// Modseq is used, and initialized if 0, for created mailboxes.

2642//

2643// Name must be in normalized form, see CheckMailboxName.

2644//

2645// Caller must hold account wlock.

2646// Caller must propagate changes if any.

2647func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, specialUse SpecialUse, modseq *ModSeq) (mb Mailbox, changes []Change, rerr error) {

2648 if norm.NFC.String(name) != name {

2649 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

2650 }

2651

2652 // Quick sanity check.

2653 if strings.EqualFold(name, "inbox") && name != "Inbox" {

2654 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

2655 }

2656

2657 // Get mailboxes with same name or prefix (parents).

2658 elems := strings.Split(name, "/")

2659 q := bstore.QueryTx[Mailbox](tx)

2660 q.FilterEqual("Expunged", false)

2661 q.FilterFn(func(xmb Mailbox) bool {

2662 t := strings.Split(xmb.Name, "/")

2663 return len(t) <= len(elems) && slices.Equal(t, elems[:len(t)])

2664 })

2665 l, err := q.List()

2666 if err != nil {

2667 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

2668 }

2669

2670 mailboxes := map[string]Mailbox{}

2671 for _, xmb := range l {

2672 mailboxes[xmb.Name] = xmb

2673 }

2674

2675 p := ""

2676 var exists bool

2677 var parentID int64

2678 for _, elem := range elems {

2679 if p != "" {

2680 p += "/"

2681 }

2682 p += elem

2683 mb, exists = mailboxes[p]

2684 if exists {

2685 parentID = mb.ID

2686 continue

2687 }

2688 uidval, err := a.NextUIDValidity(tx)

2689 if err != nil {

2690 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

2691 }

2692 if *modseq == 0 {

2693 *modseq, err = a.NextModSeq(tx)

2694 if err != nil {

2695 return Mailbox{}, nil, fmt.Errorf("next modseq: %v", err)

2696 }

2697 }

2698 mb = Mailbox{

2699 CreateSeq: *modseq,

2700 ModSeq: *modseq,

2701 ParentID: parentID,

2702 Name: p,

2703 UIDValidity: uidval,

2704 UIDNext: 1,

2705 HaveCounts: true,

2706 }

2707 err = tx.Insert(&mb)

2708 if err != nil {

2709 return Mailbox{}, nil, fmt.Errorf("creating new mailbox %q: %v", p, err)

2710 }

2711 parentID = mb.ID

2712

2713 var flags []string

2714 if subscribe {

2715 if tx.Get(&Subscription{p}) != nil {

2716 err := tx.Insert(&Subscription{p})

2717 if err != nil {

2718 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox %q: %v", p, err)

2719 }

2720 }

2721 flags = []string{`\Subscribed`}

2722 } else if err := tx.Get(&Subscription{p}); err == nil {

2723 flags = []string{`\Subscribed`}

2724 } else if err != bstore.ErrAbsent {

2725 return Mailbox{}, nil, fmt.Errorf("looking up subscription for %q: %v", p, err)

2726 }

2727

2728 changes = append(changes, ChangeAddMailbox{mb, flags})

2729 }

2730

2731 // Clear any special-use flags from existing mailboxes and assign them to this mailbox.

2732 var zeroSpecialUse SpecialUse

2733 if !exists && specialUse != zeroSpecialUse {

2734 var qerr error

2735 clearSpecialUse := func(b bool, fn func(*Mailbox) *bool) {

2736 if !b || qerr != nil {

2737 return

2738 }

2739 qs := bstore.QueryTx[Mailbox](tx)

2740 qs.FilterFn(func(xmb Mailbox) bool {

2741 return *fn(&xmb)

2742 })

2743 xmb, err := qs.Get()

2744 if err == bstore.ErrAbsent {

2745 return

2746 } else if err != nil {

2747 qerr = fmt.Errorf("looking up mailbox with special-use flag: %v", err)

2748 return

2749 }

2750 p := fn(&xmb)

2751 *p = false

2752 xmb.ModSeq = *modseq

2753 if err := tx.Update(&xmb); err != nil {

2754 qerr = fmt.Errorf("clearing special-use flag: %v", err)

2755 } else {

2756 changes = append(changes, xmb.ChangeSpecialUse())

2757 }

2758 }

2759 clearSpecialUse(specialUse.Archive, func(xmb *Mailbox) *bool { return &xmb.Archive })

2760 clearSpecialUse(specialUse.Draft, func(xmb *Mailbox) *bool { return &xmb.Draft })

2761 clearSpecialUse(specialUse.Junk, func(xmb *Mailbox) *bool { return &xmb.Junk })

2762 clearSpecialUse(specialUse.Sent, func(xmb *Mailbox) *bool { return &xmb.Sent })

2763 clearSpecialUse(specialUse.Trash, func(xmb *Mailbox) *bool { return &xmb.Trash })

2764 if qerr != nil {

2765 return Mailbox{}, nil, qerr

2766 }

2767

2768 mb.SpecialUse = specialUse

2769 mb.ModSeq = *modseq

2770 if err := tx.Update(&mb); err != nil {

2771 return Mailbox{}, nil, fmt.Errorf("setting special-use flag for new mailbox: %v", err)

2772 }

2773 changes = append(changes, mb.ChangeSpecialUse())

2774 }

2775 return mb, changes, nil

2776}

2777

2778// MailboxExists checks if mailbox exists.

2779// Caller must hold account rlock.

2780func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

2781 q := bstore.QueryTx[Mailbox](tx)

2782 q.FilterEqual("Expunged", false)

2783 q.FilterEqual("Name", name)

2784 return q.Exists()

2785}

2786

2787// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

2788func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

2789 q := bstore.QueryTx[Mailbox](tx)

2790 q.FilterEqual("Expunged", false)

2791 q.FilterEqual("Name", name)

2792 mb, err := q.Get()

2793 if err == bstore.ErrAbsent {

2794 return nil, nil

2795 }

2796 if err != nil {

2797 return nil, fmt.Errorf("looking up mailbox: %w", err)

2798 }

2799 return &mb, nil

2800}

2801

2802// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

2803// have to exist. Any parents are not automatically subscribed.

2804// Changes are returned and must be broadcasted by the caller.

2805func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

2806 if err := tx.Get(&Subscription{name}); err == nil {

2807 return nil, nil

2808 }

2809

2810 if err := tx.Insert(&Subscription{name}); err != nil {

2811 return nil, fmt.Errorf("inserting subscription: %w", err)

2812 }

2813

2814 q := bstore.QueryTx[Mailbox](tx)

2815 q.FilterEqual("Expunged", false)

2816 q.FilterEqual("Name", name)

2817 _, err := q.Get()

2818 if err == nil {

2819 return []Change{ChangeAddSubscription{name, nil}}, nil

2820 } else if err != bstore.ErrAbsent {

2821 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

2822 }

2823 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

2824}

2825

2826// MessageRuleset returns the first ruleset (if any) that matches the message

2827// represented by msgPrefix and msgFile, with smtp and validation fields from m.

2828func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

2829 if len(dest.Rulesets) == 0 {

2830 return nil

2831 }

2832

2833 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

2834 p, err := message.Parse(log.Logger, false, mr)

2835 if err != nil {

2836 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

2837 // note: part is still set.

2838 }

2839 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

2840 header, err := p.Header()

2841 if err != nil {

2842 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

2843 // todo: reject message?

2844 return nil

2845 }

2846

2847ruleset:

2848 for _, rs := range dest.Rulesets {

2849 if rs.SMTPMailFromRegexpCompiled != nil {

2850 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

2851 continue ruleset

2852 }

2853 }

2854 if rs.MsgFromRegexpCompiled != nil {

2855 if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {

2856 continue ruleset

2857 }

2858 }

2859

2860 if !rs.VerifiedDNSDomain.IsZero() {

2861 d := rs.VerifiedDNSDomain.Name()

2862 suffix := "." + d

2863 matchDomain := func(s string) bool {

2864 return s == d || strings.HasSuffix(s, suffix)

2865 }

2866 var ok bool

2867 if m.EHLOValidated && matchDomain(m.EHLODomain) {

2868 ok = true

2869 }

2870 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

2871 ok = true

2872 }

2873 for _, d := range m.DKIMDomains {

2874 if matchDomain(d) {

2875 ok = true

2876 break

2877 }

2878 }

2879 if !ok {

2880 continue ruleset

2881 }

2882 }

2883

2884 header:

2885 for _, t := range rs.HeadersRegexpCompiled {

2886 for k, vl := range header {

2887 k = strings.ToLower(k)

2888 if !t[0].MatchString(k) {

2889 continue

2890 }

2891 for _, v := range vl {

2892 v = strings.ToLower(strings.TrimSpace(v))

2893 if t[1].MatchString(v) {

2894 continue header

2895 }

2896 }

2897 }

2898 continue ruleset

2899 }

2900 return &rs

2901 }

2902 return nil

2903}

2904

2905// MessagePath returns the file system path of a message.

2906func (a *Account) MessagePath(messageID int64) string {

2907 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

2908}

2909

2910// MessageReader opens a message for reading, transparently combining the

2911// message prefix with the original incoming message.

2912func (a *Account) MessageReader(m Message) *MsgReader {

2913 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

2914}

2915

2916// DeliverDestination delivers an email to dest, based on the configured rulesets.

2917//

2918// Returns ErrOverQuota when account would be over quota after adding message.

2919//

2920// Caller must hold account wlock (mailbox may be created).

2921// Message delivery, possible mailbox creation, and updated mailbox counts are

2922// broadcasted.

2923func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

2924 var mailbox string

2925 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

2926 if rs != nil {

2927 mailbox = rs.Mailbox

2928 } else if dest.Mailbox == "" {

2929 mailbox = "Inbox"

2930 } else {

2931 mailbox = dest.Mailbox

2932 }

2933 return a.DeliverMailbox(log, mailbox, m, msgFile)

2934}

2935

2936// DeliverMailbox delivers an email to the specified mailbox.

2937//

2938// Returns ErrOverQuota when account would be over quota after adding message.

2939//

2940// Caller must hold account wlock (mailbox may be created).

2941// Message delivery, possible mailbox creation, and updated mailbox counts are

2942// broadcasted.

2943func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) (rerr error) {

2944 var changes []Change

2945

2946 var commit bool

2947 defer func() {

2948 if !commit && m.ID != 0 {

2949 p := a.MessagePath(m.ID)

2950 err := os.Remove(p)

2951 log.Check(err, "remove delivered message file", slog.String("path", p))

2952 m.ID = 0

2953 }

2954 }()

2955

2956 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2957 mb, chl, err := a.MailboxEnsure(tx, mailbox, true, SpecialUse{}, &m.ModSeq)

2958 if err != nil {

2959 return fmt.Errorf("ensuring mailbox: %w", err)

2960 }

2961 if m.CreateSeq == 0 {

2962 m.CreateSeq = m.ModSeq

2963 }

2964

2965 nmbkeywords := len(mb.Keywords)

2966

2967 if err := a.MessageAdd(log, tx, &mb, m, msgFile, AddOpts{}); err != nil {

2968 return err

2969 }

2970

2971 if err := tx.Update(&mb); err != nil {

2972 return fmt.Errorf("updating mailbox for delivery: %w", err)

2973 }

2974

2975 changes = append(changes, chl...)

2976 changes = append(changes, m.ChangeAddUID(mb), mb.ChangeCounts())

2977 if nmbkeywords != len(mb.Keywords) {

2978 changes = append(changes, mb.ChangeKeywords())

2979 }

2980 return nil

2981 })

2982 if err != nil {

2983 return err

2984 }

2985 commit = true

2986 BroadcastChanges(a, changes)

2987 return nil

2988}

2989

2990type RemoveOpts struct {

2991 JunkFilter *junk.Filter // If set, this filter is used for training, instead of opening and saving the junk filter.

2992}

2993

2994// MessageRemove markes messages as expunged, updates mailbox counts for the

2995// messages, sets a new modseq on the messages and mailbox, untrains the junk

2996// filter and queues the messages for erasing when the last reference has gone.

2997//

2998// Caller must save the modified mailbox to the database.

2999//

3000// The disk usage is not immediately updated. That will happen when the message

3001// is actually removed from disk.

3002//

3003// The junk filter is untrained for the messages if it was trained.

3004// Useful as optimization when messages are moved and the junk/nonjunk flags do not

3005// change (which can happen due to automatic junk/nonjunk flags for mailboxes).

3006//

3007// An empty list of messages results in an error.

3008//

3009// Caller must broadcast changes.

3010//

3011// Must be called with wlock held.

3012func (a *Account) MessageRemove(log mlog.Log, tx *bstore.Tx, modseq ModSeq, mb *Mailbox, opts RemoveOpts, l ...Message) (chremuids ChangeRemoveUIDs, chmbc ChangeMailboxCounts, rerr error) {

3013 if len(l) == 0 {

3014 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("must expunge at least one message")

3015 }

3016

3017 mb.ModSeq = modseq

3018

3019 // Remove any message recipients.

3020 anyIDs := make([]any, len(l))

3021 for i, m := range l {

3022 anyIDs[i] = m.ID

3023 }

3024 qmr := bstore.QueryTx[Recipient](tx)

3025 qmr.FilterEqual("MessageID", anyIDs...)

3026 if _, err := qmr.Delete(); err != nil {

3027 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("deleting message recipients for messages: %w", err)

3028 }

3029

3030 // Loaded lazily.

3031 jf := opts.JunkFilter

3032

3033 // Mark messages expunged.

3034 ids := make([]int64, 0, len(l))

3035 uids := make([]UID, 0, len(l))

3036 for _, m := range l {

3037 ids = append(ids, m.ID)

3038 uids = append(uids, m.UID)

3039

3040 if m.Expunged {

3041 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("message %d is already expunged", m.ID)

3042 }

3043

3044 mb.Sub(m.MailboxCounts())

3045

3046 m.ModSeq = modseq

3047 m.Expunged = true

3048 m.Junk = false

3049 m.Notjunk = false

3050

3051 if err := tx.Update(&m); err != nil {

3052 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("marking message %d expunged: %v", m.ID, err)

3053 }

3054

3055 // Ensure message gets erased in future.

3056 if err := tx.Insert(&MessageErase{m.ID, false}); err != nil {

3057 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("inserting message erase %d : %v", m.ID, err)

3058 }

3059

3060 if m.TrainedJunk == nil || !a.HasJunkFilter() {

3061 continue

3062 }

3063 // Untrain, as needed by updated flags Junk/Notjunk to false.

3064 if jf == nil {

3065 var err error

3066 jf, _, err = a.OpenJunkFilter(context.TODO(), log)

3067 if err != nil {

3068 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("open junk filter: %v", err)

3069 }

3070 defer func() {

3071 err := jf.Close()

3072 if rerr == nil {

3073 rerr = err

3074 } else {

3075 log.Check(err, "closing junk filter")

3076 }

3077 }()

3078 }

3079 if err := a.RetrainMessage(context.TODO(), log, tx, jf, &m); err != nil {

3080 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("retraining expunged messages: %w", err)

3081 }

3082 }

3083

3084 return ChangeRemoveUIDs{mb.ID, uids, modseq, ids, mb.UIDNext, mb.MessageCountIMAP(), uint32(mb.MailboxCounts.Unseen)}, mb.ChangeCounts(), nil

3085}

3086

3087// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

3088//

3089// The changed mailbox is saved to the database.

3090//

3091// Caller most hold account wlock.

3092// Caller must broadcast changes.

3093func (a *Account) TidyRejectsMailbox(log mlog.Log, tx *bstore.Tx, mbRej *Mailbox) (changes []Change, hasSpace bool, rerr error) {

3094 // Gather old messages to expunge.

3095 old := time.Now().Add(-14 * 24 * time.Hour)

3096 qdel := bstore.QueryTx[Message](tx)

3097 qdel.FilterNonzero(Message{MailboxID: mbRej.ID})

3098 qdel.FilterEqual("Expunged", false)

3099 qdel.FilterLess("Received", old)

3100 qdel.SortAsc("UID")

3101 expunge, err := qdel.List()

3102 if err != nil {

3103 return nil, false, fmt.Errorf("listing old messages: %w", err)

3104 }

3105

3106 if len(expunge) > 0 {

3107 modseq, err := a.NextModSeq(tx)

3108 if err != nil {

3109 return nil, false, fmt.Errorf("next mod seq: %v", err)

3110 }

3111

3112 chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mbRej, RemoveOpts{}, expunge...)

3113 if err != nil {

3114 return nil, false, fmt.Errorf("removing messages: %w", err)

3115 }

3116 if err := tx.Update(mbRej); err != nil {

3117 return nil, false, fmt.Errorf("updating mailbox: %v", err)

3118 }

3119 changes = append(changes, chremuids, chmbcounts)

3120 }

3121

3122 // We allow up to n messages.

3123 qcount := bstore.QueryTx[Message](tx)

3124 qcount.FilterNonzero(Message{MailboxID: mbRej.ID})

3125 qcount.FilterEqual("Expunged", false)

3126 qcount.Limit(1000)

3127 n, err := qcount.Count()

3128 if err != nil {

3129 return nil, false, fmt.Errorf("counting rejects: %w", err)

3130 }

3131 hasSpace = n < 1000

3132

3133 return changes, hasSpace, nil

3134}

3135

3136// RejectsRemove removes a message from the rejects mailbox if present.

3137//

3138// Caller most hold account wlock.

3139// Changes are broadcasted.

3140func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

3141 var changes []Change

3142

3143 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

3144 mb, err := a.MailboxFind(tx, rejectsMailbox)

3145 if err != nil {

3146 return fmt.Errorf("finding mailbox: %w", err)

3147 }

3148 if mb == nil {

3149 return nil

3150 }

3151

3152 q := bstore.QueryTx[Message](tx)

3153 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

3154 q.FilterEqual("Expunged", false)

3155 expunge, err := q.List()

3156 if err != nil {

3157 return fmt.Errorf("listing messages to remove: %w", err)

3158 }

3159

3160 if len(expunge) == 0 {

3161 return nil

3162 }

3163

3164 modseq, err := a.NextModSeq(tx)

3165 if err != nil {

3166 return fmt.Errorf("get next mod seq: %v", err)

3167 }

3168

3169 chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mb, RemoveOpts{}, expunge...)

3170 if err != nil {

3171 return fmt.Errorf("removing messages: %w", err)

3172 }

3173 changes = append(changes, chremuids, chmbcounts)

3174

3175 if err := tx.Update(mb); err != nil {

3176 return fmt.Errorf("saving mailbox: %w", err)

3177 }

3178

3179 return nil

3180 })

3181 if err != nil {

3182 return err

3183 }

3184

3185 BroadcastChanges(a, changes)

3186

3187 return nil

3188}

3189

3190// AddMessageSize adjusts the DiskUsage.MessageSize by size.

3191func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

3192 du := DiskUsage{ID: 1}

3193 if err := tx.Get(&du); err != nil {

3194 return fmt.Errorf("get diskusage: %v", err)

3195 }

3196 du.MessageSize += size

3197 if du.MessageSize < 0 {

3198 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

3199 }

3200 if err := tx.Update(&du); err != nil {

3201 return fmt.Errorf("update total message size: %v", err)

3202 }

3203 return nil

3204}

3205

3206// QuotaMessageSize returns the effective maximum total message size for an

3207// account. Returns 0 if there is no maximum.

3208func (a *Account) QuotaMessageSize() int64 {

3209 conf, _ := a.Conf()

3210 size := conf.QuotaMessageSize

3211 if size == 0 {

3212 size = mox.Conf.Static.QuotaMessageSize

3213 }

3214 if size < 0 {

3215 size = 0

3216 }

3217 return size

3218}

3219

3220// CanAddMessageSize checks if a message of size bytes can be added, depending on

3221// total message size and configured quota for account.

3222func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

3223 maxSize = a.QuotaMessageSize()

3224 if maxSize <= 0 {

3225 return true, 0, nil

3226 }

3227

3228 du := DiskUsage{ID: 1}

3229 if err := tx.Get(&du); err != nil {

3230 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

3231 }

3232 return du.MessageSize+size <= maxSize, maxSize, nil

3233}

3234

3235// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

3236var authCache = struct {

3237 sync.Mutex

3238 success map[authKey]string

3239}{

3240 success: map[authKey]string{},

3241}

3242

3243type authKey struct {

3244 email, hash string

3245}

3246

3247// StartAuthCache starts a goroutine that regularly clears the auth cache.

3248func StartAuthCache() {

3249 go manageAuthCache()

3250}

3251

3252func manageAuthCache() {

3253 for {

3254 authCache.Lock()

3255 authCache.success = map[authKey]string{}

3256 authCache.Unlock()

3257 time.Sleep(15 * time.Minute)

3258 }

3259}

3260

3261// OpenEmailAuth opens an account given an email address and password.

3262//

3263// The email address may contain a catchall separator.

3264// For invalid credentials, a nil account is returned, but accName may be

3265// non-empty.

3266func OpenEmailAuth(log mlog.Log, email string, password string, checkLoginDisabled bool) (racc *Account, raccName string, rerr error) {

3267 // We check for LoginDisabled after verifying the password. Otherwise users can get

3268 // messages about the account being disabled without knowing the password.

3269 acc, accName, _, err := OpenEmail(log, email, false)

3270 if err != nil {

3271 return nil, "", err

3272 }

3273

3274 defer func() {

3275 if rerr != nil {

3276 err := acc.Close()

3277 log.Check(err, "closing account after open auth failure")

3278 acc = nil

3279 }

3280 }()

3281

3282 password, err = precis.OpaqueString.String(password)

3283 if err != nil {

3284 return nil, "", ErrUnknownCredentials

3285 }

3286

3287 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

3288 if err != nil {

3289 if err == bstore.ErrAbsent {

3290 return nil, "", ErrUnknownCredentials

3291 }

3292 return nil, "", fmt.Errorf("looking up password: %v", err)

3293 }

3294 authCache.Lock()

3295 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

3296 authCache.Unlock()

3297 if !ok {

3298 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

3299 return nil, "", ErrUnknownCredentials

3300 }

3301 }

3302 if checkLoginDisabled {

3303 conf, aok := acc.Conf()

3304 if !aok {

3305 return nil, "", fmt.Errorf("cannot find config for account")

3306 } else if conf.LoginDisabled != "" {

3307 return nil, "", fmt.Errorf("%w: %s", ErrLoginDisabled, conf.LoginDisabled)

3308 }

3309 }

3310 authCache.Lock()

3311 authCache.success[authKey{email, pw.Hash}] = password

3312 authCache.Unlock()

3313 return acc, accName, nil

3314}

3315

3316// OpenEmail opens an account given an email address.

3317//

3318// The email address may contain a catchall separator.

3319//

3320// Returns account on success, may return non-empty account name even on error.

3321func OpenEmail(log mlog.Log, email string, checkLoginDisabled bool) (*Account, string, config.Destination, error) {

3322 addr, err := smtp.ParseAddress(email)

3323 if err != nil {

3324 return nil, "", config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

3325 }

3326 accountName, _, _, dest, err := mox.LookupAddress(addr.Localpart, addr.Domain, false, false, false)

3327 if err != nil && (errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

3328 return nil, accountName, config.Destination{}, ErrUnknownCredentials

3329 } else if err != nil {

3330 return nil, accountName, config.Destination{}, fmt.Errorf("looking up address: %v", err)

3331 }

3332 acc, err := OpenAccount(log, accountName, checkLoginDisabled)

3333 if err != nil {

3334 return nil, accountName, config.Destination{}, err

3335 }

3336 return acc, accountName, dest, nil

3337}

3338

3339// We store max 1<<shift files in each subdir of an account "msg" directory.

3340// Defaults to 1 for easy use in tests. Set to 13, for 8k message files, in main

3341// for normal operation.

3342var msgFilesPerDirShift = 1

3343var msgFilesPerDir int64 = 1 << msgFilesPerDirShift

3344

3345func MsgFilesPerDirShiftSet(shift int) {

3346 msgFilesPerDirShift = shift

3347 msgFilesPerDir = 1 << shift

3348}

3349

3350// 64 characters, must be power of 2 for MessagePath

3351const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

3352

3353// MessagePath returns the filename of the on-disk filename, relative to the

3354// containing directory such as <account>/msg or queue.

3355// Returns names like "AB/1".

3356func MessagePath(messageID int64) string {

3357 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

3358}

3359

3360// messagePathElems returns the elems, for a single join without intermediate

3361// string allocations.

3362func messagePathElems(messageID int64) []string {

3363 v := messageID >> msgFilesPerDirShift

3364 dir := ""

3365 for {

3366 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

3367 v >>= 6

3368 if v == 0 {

3369 break

3370 }

3371 }

3372 return []string{dir, strconv.FormatInt(messageID, 10)}

3373}

3374

3375// Set returns a copy of f, with each flag that is true in mask set to the

3376// value from flags.

3377func (f Flags) Set(mask, flags Flags) Flags {

3378 set := func(d *bool, m, v bool) {

3379 if m {

3380 *d = v

3381 }

3382 }

3383 r := f

3384 set(&r.Seen, mask.Seen, flags.Seen)

3385 set(&r.Answered, mask.Answered, flags.Answered)

3386 set(&r.Flagged, mask.Flagged, flags.Flagged)

3387 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

3388 set(&r.Junk, mask.Junk, flags.Junk)

3389 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

3390 set(&r.Deleted, mask.Deleted, flags.Deleted)

3391 set(&r.Draft, mask.Draft, flags.Draft)

3392 set(&r.Phishing, mask.Phishing, flags.Phishing)

3393 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

3394 return r

3395}

3396

3397// Changed returns a mask of flags that have been between f and other.

3398func (f Flags) Changed(other Flags) (mask Flags) {

3399 mask.Seen = f.Seen != other.Seen

3400 mask.Answered = f.Answered != other.Answered

3401 mask.Flagged = f.Flagged != other.Flagged

3402 mask.Forwarded = f.Forwarded != other.Forwarded

3403 mask.Junk = f.Junk != other.Junk

3404 mask.Notjunk = f.Notjunk != other.Notjunk

3405 mask.Deleted = f.Deleted != other.Deleted

3406 mask.Draft = f.Draft != other.Draft

3407 mask.Phishing = f.Phishing != other.Phishing

3408 mask.MDNSent = f.MDNSent != other.MDNSent

3409 return

3410}

3411

3412// Strings returns the flags that are set in their string form.

3413func (f Flags) Strings() []string {

3414 fields := []struct {

3415 word string

3416 have bool

3417 }{

3418 {`$forwarded`, f.Forwarded},

3419 {`$junk`, f.Junk},

3420 {`$mdnsent`, f.MDNSent},

3421 {`$notjunk`, f.Notjunk},

3422 {`$phishing`, f.Phishing},

3423 {`\answered`, f.Answered},

3424 {`\deleted`, f.Deleted},

3425 {`\draft`, f.Draft},

3426 {`\flagged`, f.Flagged},

3427 {`\seen`, f.Seen},

3428 }

3429 var l []string

3430 for _, fh := range fields {

3431 if fh.have {

3432 l = append(l, fh.word)

3433 }

3434 }

3435 return l

3436}

3437

3438var systemWellKnownFlags = map[string]bool{

3439 `\answered`: true,

3440 `\flagged`: true,

3441 `\deleted`: true,

3442 `\seen`: true,

3443 `\draft`: true,

3444 `$junk`: true,

3445 `$notjunk`: true,

3446 `$forwarded`: true,

3447 `$phishing`: true,

3448 `$mdnsent`: true,

3449}

3450

3451// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

3452// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

3453func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

3454 fields := map[string]*bool{

3455 `\answered`: &flags.Answered,

3456 `\flagged`: &flags.Flagged,

3457 `\deleted`: &flags.Deleted,

3458 `\seen`: &flags.Seen,

3459 `\draft`: &flags.Draft,

3460 `$junk`: &flags.Junk,

3461 `$notjunk`: &flags.Notjunk,

3462 `$forwarded`: &flags.Forwarded,

3463 `$phishing`: &flags.Phishing,

3464 `$mdnsent`: &flags.MDNSent,

3465 }

3466 seen := map[string]bool{}

3467 for _, f := range l {

3468 f = strings.ToLower(f)

3469 if field, ok := fields[f]; ok {

3470 *field = true

3471 } else if seen[f] {

3472 if mox.Pedantic {

3473 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

3474 }

3475 } else {

3476 if err := CheckKeyword(f); err != nil {

3477 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

3478 }

3479 keywords = append(keywords, f)

3480 seen[f] = true

3481 }

3482 }

3483 sort.Strings(keywords)

3484 return flags, keywords, nil

3485}

3486

3487// RemoveKeywords removes keywords from l, returning whether any modifications were

3488// made, and a slice, a new slice in case of modifications. Keywords must have been

3489// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

3490// be used with valid keywords, not with system flags like \Seen.

3491func RemoveKeywords(l, remove []string) ([]string, bool) {

3492 var copied bool

3493 var changed bool

3494 for _, k := range remove {

3495 if i := slices.Index(l, k); i >= 0 {

3496 if !copied {

3497 l = slices.Clone(l)

3498 copied = true

3499 }

3500 copy(l[i:], l[i+1:])

3501 l = l[:len(l)-1]

3502 changed = true

3503 }

3504 }

3505 return l, changed

3506}

3507

3508// MergeKeywords adds keywords from add into l, returning whether it added any

3509// keyword, and the slice with keywords, a new slice if modifications were made.

3510// Keywords are only added if they aren't already present. Should only be used with

3511// keywords, not with system flags like \Seen.

3512func MergeKeywords(l, add []string) ([]string, bool) {

3513 var copied bool

3514 var changed bool

3515 for _, k := range add {

3516 if !slices.Contains(l, k) {

3517 if !copied {

3518 l = slices.Clone(l)

3519 copied = true

3520 }

3521 l = append(l, k)

3522 changed = true

3523 }

3524 }

3525 if changed {

3526 sort.Strings(l)

3527 }

3528 return l, changed

3529}

3530

3531// CheckKeyword returns an error if kw is not a valid keyword. Kw should

3532// already be in lower-case.

3533func CheckKeyword(kw string) error {

3534 if kw == "" {

3535 return fmt.Errorf("keyword cannot be empty")

3536 }

3537 if systemWellKnownFlags[kw] {

3538 return fmt.Errorf("cannot use well-known flag as keyword")

3539 }

3540 for _, c := range kw {

3541 // ../rfc/9051:6334

3542 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

3543 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

3544 }

3545 }

3546 return nil

3547}

3548

3549// SendLimitReached checks whether sending a message to recipients would reach

3550// the limit of outgoing messages for the account. If so, the message should

3551// not be sent. If the returned numbers are >= 0, the limit was reached and the

3552// values are the configured limits.

3553//

3554// To limit damage to the internet and our reputation in case of account

3555// compromise, we limit the max number of messages sent in a 24 hour window, both

3556// total number of messages and number of first-time recipients.

3557func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

3558 conf, _ := a.Conf()

3559 msgmax := conf.MaxOutgoingMessagesPerDay

3560 if msgmax == 0 {

3561 // For human senders, 1000 recipients in a day is quite a lot.

3562 msgmax = 1000

3563 }

3564 rcptmax := conf.MaxFirstTimeRecipientsPerDay

3565 if rcptmax == 0 {

3566 // Human senders may address a new human-sized list of people once in a while. In

3567 // case of a compromise, a spammer will probably try to send to many new addresses.

3568 rcptmax = 200

3569 }

3570

3571 rcpts := map[string]time.Time{}

3572 n := 0

3573 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

3574 n++

3575 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

3576 rcpts[o.Recipient] = o.Submitted

3577 }

3578 return nil

3579 })

3580 if err != nil {

3581 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

3582 }

3583 if n+len(recipients) > msgmax {

3584 return msgmax, -1, nil

3585 }

3586

3587 // Only check if max first-time recipients is reached if there are enough messages

3588 // to trigger the limit.

3589 if n+len(recipients) < rcptmax {

3590 return -1, -1, nil

3591 }

3592

3593 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

3594 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

3595 return !exists, err

3596 }

3597

3598 firsttime := 0

3599 now := time.Now()

3600 for _, r := range recipients {

3601 if first, err := isFirstTime(r.XString(true), now); err != nil {

3602 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

3603 } else if first {

3604 firsttime++

3605 }

3606 }

3607 for r, t := range rcpts {

3608 if first, err := isFirstTime(r, t); err != nil {

3609 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

3610 } else if first {

3611 firsttime++

3612 }

3613 }

3614 if firsttime > rcptmax {

3615 return -1, rcptmax, nil

3616 }

3617 return -1, -1, nil

3618}

3619

3620var ErrMailboxExpunged = errors.New("mailbox was deleted")

3621

3622// MailboxID gets a mailbox by ID.

3623//

3624// Returns bstore.ErrAbsent if the mailbox does not exist.

3625// Returns ErrMailboxExpunged if the mailbox is expunged.

3626func MailboxID(tx *bstore.Tx, id int64) (Mailbox, error) {

3627 mb := Mailbox{ID: id}

3628 err := tx.Get(&mb)

3629 if err == nil && mb.Expunged {

3630 return Mailbox{}, ErrMailboxExpunged

3631 }

3632 return mb, err

3633}

3634

3635// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

3636// the total list of created mailboxes is returned in created. On success, if

3637// exists is false and rerr nil, the changes must be broadcasted by the caller.

3638//

3639// The mailbox is created with special-use flags, with those flags taken away from

3640// other mailboxes if they have them, reflected in the returned changes.

3641//

3642// Name must be in normalized form, see CheckMailboxName.

3643func (a *Account) MailboxCreate(tx *bstore.Tx, name string, specialUse SpecialUse) (nmb Mailbox, changes []Change, created []string, exists bool, rerr error) {

3644 elems := strings.Split(name, "/")

3645 var p string

3646 var modseq ModSeq

3647 for i, elem := range elems {

3648 if i > 0 {

3649 p += "/"

3650 }

3651 p += elem

3652 exists, err := a.MailboxExists(tx, p)

3653 if err != nil {

3654 return Mailbox{}, nil, nil, false, fmt.Errorf("checking if mailbox exists")

3655 }

3656 if exists {

3657 if i == len(elems)-1 {

3658 return Mailbox{}, nil, nil, true, fmt.Errorf("mailbox already exists")

3659 }

3660 continue

3661 }

3662 mb, nchanges, err := a.MailboxEnsure(tx, p, true, specialUse, &modseq)

3663 if err != nil {

3664 return Mailbox{}, nil, nil, false, fmt.Errorf("ensuring mailbox exists: %v", err)

3665 }

3666 nmb = mb

3667 changes = append(changes, nchanges...)

3668 created = append(created, p)

3669 }

3670 return nmb, changes, created, false, nil

3671}

3672

3673// MailboxRename renames mailbox mbsrc to dst, including children of mbsrc, and

3674// adds missing parents for dst.

3675//

3676// Name must be in normalized form, see CheckMailboxName, and cannot be Inbox.

3677func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc *Mailbox, dst string, modseq *ModSeq) (changes []Change, isInbox, alreadyExists bool, rerr error) {

3678 if mbsrc.Name == "Inbox" || dst == "Inbox" {

3679 return nil, true, false, fmt.Errorf("inbox cannot be renamed")

3680 }

3681

3682 // Check if destination mailbox already exists.

3683 if exists, err := a.MailboxExists(tx, dst); err != nil {

3684 return nil, false, false, fmt.Errorf("checking if destination mailbox exists: %v", err)

3685 } else if exists {

3686 return nil, false, true, fmt.Errorf("destination mailbox already exists")

3687 }

3688

3689 if *modseq == 0 {