1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

5

6Layout of storage for accounts:

7

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

10

11Index.db holds tables for user information, mailboxes, and messages. Message contents

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

21

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

23

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "io/fs"

37 "log/slog"

38 "os"

39 "path/filepath"

40 "reflect"

41 "runtime/debug"

42 "slices"

43 "sort"

44 "strconv"

45 "strings"

46 "sync"

47 "time"

48

49 "golang.org/x/crypto/bcrypt"

50 "golang.org/x/text/secure/precis"

51 "golang.org/x/text/unicode/norm"

52

53 "github.com/mjl-/bstore"

54

55 "github.com/mjl-/mox/config"

56 "github.com/mjl-/mox/dns"

57 "github.com/mjl-/mox/junk"

58 "github.com/mjl-/mox/message"

59 "github.com/mjl-/mox/metrics"

60 "github.com/mjl-/mox/mlog"

61 "github.com/mjl-/mox/mox-"

62 "github.com/mjl-/mox/moxio"

63 "github.com/mjl-/mox/moxvar"

64 "github.com/mjl-/mox/publicsuffix"

65 "github.com/mjl-/mox/scram"

66 "github.com/mjl-/mox/smtp"

67)

68

69// If true, each time an account is closed its database file is checked for

70// consistency. If an inconsistency is found, panic is called. Set by default

71// because of all the packages with tests, the mox main function sets it to

72// false again.

73var CheckConsistencyOnClose = true

74

75var (

76 ErrUnknownMailbox = errors.New("no such mailbox")

77 ErrUnknownCredentials = errors.New("credentials not found")

78 ErrAccountUnknown = errors.New("no such account")

79 ErrOverQuota = errors.New("account over quota")

80 ErrLoginDisabled = errors.New("login disabled for account")

81)

82

83var DefaultInitialMailboxes = config.InitialMailboxes{

84 SpecialUse: config.SpecialUseMailboxes{

85 Sent: "Sent",

86 Archive: "Archive",

87 Trash: "Trash",

88 Draft: "Drafts",

89 Junk: "Junk",

90 },

91}

92

93type SCRAM struct {

94 Salt []byte

95 Iterations int

96 SaltedPassword []byte

97}

98

99// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

100// block with (a derivation of) the key/password, so we don't store the password in plain

101// text.

102type CRAMMD5 struct {

103 Ipad hash.Hash

104 Opad hash.Hash

105}

106

107// BinaryMarshal is used by bstore to store the ipad/opad hash states.

108func (c CRAMMD5) MarshalBinary() ([]byte, error) {

109 if c.Ipad == nil || c.Opad == nil {

110 return nil, nil

111 }

112

113 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

114 if err != nil {

115 return nil, fmt.Errorf("marshal ipad: %v", err)

116 }

117 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

118 if err != nil {

119 return nil, fmt.Errorf("marshal opad: %v", err)

120 }

121 buf := make([]byte, 2+len(ipad)+len(opad))

122 ipadlen := uint16(len(ipad))

123 buf[0] = byte(ipadlen >> 8)

124 buf[1] = byte(ipadlen >> 0)

125 copy(buf[2:], ipad)

126 copy(buf[2+len(ipad):], opad)

127 return buf, nil

128}

129

130// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

131func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

132 if len(buf) == 0 {

133 *c = CRAMMD5{}

134 return nil

135 }

136 if len(buf) < 2 {

137 return fmt.Errorf("short buffer")

138 }

139 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

140 if len(buf) < 2+ipadlen {

141 return fmt.Errorf("buffer too short for ipadlen")

142 }

143 ipad := md5.New()

144 opad := md5.New()

145 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

146 return fmt.Errorf("unmarshal ipad: %v", err)

147 }

148 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

149 return fmt.Errorf("unmarshal opad: %v", err)

150 }

151 *c = CRAMMD5{ipad, opad}

152 return nil

153}

154

155// Password holds credentials in various forms, for logging in with SMTP/IMAP.

156type Password struct {

157 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

158 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

159 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

160 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

161}

162

163// Subjectpass holds the secret key used to sign subjectpass tokens.

164type Subjectpass struct {

165 Email string // Our destination address (canonical, with catchall localpart stripped).

166 Key string

167}

168

169// NextUIDValidity is a singleton record in the database with the next UIDValidity

170// to use for the next mailbox.

171type NextUIDValidity struct {

172 ID int // Just a single record with ID 1.

173 Next uint32

174}

175

176// SyncState track ModSeqs.

177type SyncState struct {

178 ID int // Just a single record with ID 1.

179

180 // Last used, next assigned will be one higher. The first value we hand out is 2.

181 // That's because 0 (the default value for old existing messages, from before the

182 // Message.ModSeq field) is special in IMAP, so we return it as 1.

183 LastModSeq ModSeq `bstore:"nonzero"`

184

185 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

186 // and requests changes based on a modseq before this one, we don't have the

187 // history to provide information about deletions. We normally keep these expunged

188 // records around, but we may periodically truly delete them to reclaim storage

189 // space. Initially set to -1 because we don't want to match with any ModSeq in the

190 // database, which can be zero values.

191 HighestDeletedModSeq ModSeq

192}

193

194// Mailbox is collection of messages, e.g. Inbox or Sent.

195type Mailbox struct {

196 ID int64

197

198 CreateSeq ModSeq

199 ModSeq ModSeq `bstore:"index"` // Of last change, or when deleted.

200 Expunged bool

201

202 ParentID int64 `bstore:"ref Mailbox"` // Zero for top-level mailbox.

203

204 // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy.

205 // Names must be unique for mailboxes that are not expunged.

206 Name string `bstore:"nonzero"`

207

208 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

209 // name, UIDValidity must be changed. Used by IMAP for synchronization.

210 UIDValidity uint32

211

212 // UID likely to be assigned to next message. Used by IMAP to detect messages

213 // delivered to a mailbox.

214 UIDNext UID

215

216 SpecialUse

217

218 // Keywords as used in messages. Storing a non-system keyword for a message

219 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

220 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

221 // lower case (for JMAP), sorted.

222 Keywords []string

223

224 HaveCounts bool // Deprecated. Covered by Upgrade.MailboxCounts. No longer read.

225 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

226}

227

228// Annotation is a per-mailbox or global (per-account) annotation for the IMAP

229// metadata extension, currently always a private annotation.

230type Annotation struct {

231 ID int64

232

233 CreateSeq ModSeq

234 ModSeq ModSeq `bstore:"index"`

235 Expunged bool

236

237 // Can be zero, indicates global (per-account) annotation.

238 MailboxID int64 `bstore:"ref Mailbox,index MailboxID+Key"`

239

240 // "Entry name", always starts with "/private/" or "/shared/". Stored lower-case,

241 // comparisons must be done case-insensitively.

242 Key string `bstore:"nonzero"`

243

244 IsString bool // If true, the value is a string instead of bytes.

245 Value []byte

246}

247

248// Change returns a broadcastable change for the annotation.

249func (a Annotation) Change(mailboxName string) ChangeAnnotation {

250 return ChangeAnnotation{a.MailboxID, mailboxName, a.Key, a.ModSeq}

251}

252

253// MailboxCounts tracks statistics about messages for a mailbox.

254type MailboxCounts struct {

255 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

256 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

257 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

258 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

259 Size int64 // Number of bytes for all messages.

260}

261

262// MessageCountIMAP returns the total message count for use in IMAP. In IMAP,

263// message marked \Deleted are included, in JMAP they those messages are not

264// visible at all.

265func (mc MailboxCounts) MessageCountIMAP() uint32 {

266 return uint32(mc.Total + mc.Deleted)

267}

268

269func (mc MailboxCounts) String() string {

270 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

271}

272

273// Add increases mailbox counts mc with those of delta.

274func (mc *MailboxCounts) Add(delta MailboxCounts) {

275 mc.Total += delta.Total

276 mc.Deleted += delta.Deleted

277 mc.Unread += delta.Unread

278 mc.Unseen += delta.Unseen

279 mc.Size += delta.Size

280}

281

282// Add decreases mailbox counts mc with those of delta.

283func (mc *MailboxCounts) Sub(delta MailboxCounts) {

284 mc.Total -= delta.Total

285 mc.Deleted -= delta.Deleted

286 mc.Unread -= delta.Unread

287 mc.Unseen -= delta.Unseen

288 mc.Size -= delta.Size

289}

290

291// SpecialUse identifies a specific role for a mailbox, used by clients to

292// understand where messages should go.

293type SpecialUse struct {

294 Archive bool

295 Draft bool // "Drafts"

296 Junk bool

297 Sent bool

298 Trash bool

299}

300

301// UIDNextAdd increases the UIDNext value by n, returning an error on overflow.

302func (mb *Mailbox) UIDNextAdd(n int) error {

303 uidnext := mb.UIDNext + UID(n)

304 if uidnext < mb.UIDNext {

305 return fmt.Errorf("uid overflow on mailbox %q (id %d): uidnext %d, adding %d; consider recreating the mailbox and copying its messages to compact", mb.Name, mb.ID, mb.UIDNext, n)

306 }

307 mb.UIDNext = uidnext

308 return nil

309}

310

311// CalculateCounts calculates the full current counts for messages in the mailbox.

312func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

313 q := bstore.QueryTx[Message](tx)

314 q.FilterNonzero(Message{MailboxID: mb.ID})

315 q.FilterEqual("Expunged", false)

316 err = q.ForEach(func(m Message) error {

317 mc.Add(m.MailboxCounts())

318 return nil

319 })

320 return

321}

322

323// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

324// other connections.

325func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

326 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse, mb.ModSeq}

327}

328

329// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

330// setting a new keyword on a message in the mailbox), for broadcasting to other

331// connections.

332func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

333 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

334}

335

336func (mb Mailbox) ChangeAddMailbox(flags []string) ChangeAddMailbox {

337 return ChangeAddMailbox{Mailbox: mb, Flags: flags}

338}

339

340func (mb Mailbox) ChangeRemoveMailbox() ChangeRemoveMailbox {

341 return ChangeRemoveMailbox{mb.ID, mb.Name, mb.ModSeq}

342}

343

344// KeywordsChanged returns whether the keywords in a mailbox have changed.

345func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

346 if len(mb.Keywords) != len(origmb.Keywords) {

347 return true

348 }

349 // Keywords are stored sorted.

350 for i, kw := range mb.Keywords {

351 if origmb.Keywords[i] != kw {

352 return true

353 }

354 }

355 return false

356}

357

358// CountsChange returns a change with mailbox counts.

359func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

360 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

361}

362

363// Subscriptions are separate from existence of mailboxes.

364type Subscription struct {

365 Name string

366}

367

368// Flags for a mail message.

369type Flags struct {

370 Seen bool

371 Answered bool

372 Flagged bool

373 Forwarded bool

374 Junk bool

375 Notjunk bool

376 Deleted bool

377 Draft bool

378 Phishing bool

379 MDNSent bool

380}

381

382// FlagsAll is all flags set, for use as mask.

383var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

384

385// Validation of "message From" domain.

386type Validation uint8

387

388const (

389 ValidationUnknown Validation = 0

390 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

391 ValidationDMARC Validation = 2 // Actual DMARC policy.

392 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

393 ValidationPass Validation = 4 // For SPF.

394 ValidationNeutral Validation = 5 // For SPF.

395 ValidationTemperror Validation = 6

396 ValidationPermerror Validation = 7

397 ValidationFail Validation = 8

398 ValidationSoftfail Validation = 9 // For SPF.

399 ValidationNone Validation = 10 // E.g. No records.

400)

401

402// Message stored in database and per-message file on disk.

403//

404// Contents are always the combined data from MsgPrefix and the on-disk file named

405// based on ID.

406//

407// Messages always have a header section, even if empty. Incoming messages without

408// header section must get an empty header section added before inserting.

409type Message struct {

410 // ID of the message, determines path to on-disk message file. Set when adding to a

411 // mailbox. When a message is moved to another mailbox, the mailbox ID is changed,

412 // but for synchronization purposes, a new Message record is inserted (which gets a

413 // new ID) with the Expunged field set and the MailboxID and UID copied.

414 ID int64

415

416 // UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per

417 // mailbox. The UID of a message can never change (though messages can be copied),

418 // and the contents of a message/UID also never changes.

419 UID UID `bstore:"nonzero"`

420

421 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

422

423 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

424 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

425 // always <= ModSeq. If Expunged is set, the message has been removed and should not

426 // be returned to the user. In this case, ModSeq is the Seq where the message is

427 // removed, and will never be changed again.

428 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

429 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

430 // The index on CreateSeq helps efficiently finding created messages for JMAP.

431 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

432 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

433 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

434 // return modseq to clients, we turn 0 into 1.

435 ModSeq ModSeq `bstore:"index"`

436 CreateSeq ModSeq `bstore:"index"`

437 Expunged bool

438

439 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

440 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

441 // flag cleared.

442 IsReject bool

443

444 // If set, this is a forwarded message (through a ruleset with IsForward). This

445 // causes fields used during junk analysis to be moved to their Orig variants, and

446 // masked IP fields cleared, so they aren't used in junk classifications for

447 // incoming messages. This ensures the forwarded messages don't cause negative

448 // reputation for the forwarding mail server, which may also be sending regular

449 // messages.

450 IsForward bool

451

452 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

453 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

454 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

455 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

456 // per-mailbox reputation.

457 //

458 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

459 // mailbox or diverted to the Introbox, it is set to the intended mailbox according

460 // to delivery rules, typically that of Inbox. When such a message is moved to its

461 // intended mailbox, MailboxOrigID is corrected by setting it to MailboxDestinedID.

462 // This ensures the message is used for reputation calculation for future

463 // deliveries to that mailbox.

464 //

465 // These are not bstore references to prevent having to update all messages in a

466 // mailbox when the original mailbox is removed. Use of these fields requires

467 // checking if the mailbox still exists.

468 MailboxOrigID int64

469 MailboxDestinedID int64

470

471 // Received indicates time of receival over SMTP, or of IMAP APPEND.

472 Received time.Time `bstore:"default now,index"`

473

474 // SaveDate is the time of copy/move/save to a mailbox, used with IMAP SAVEDATE

475 // extension. Must be updated each time a message is copied/moved to another

476 // mailbox. Can be nil for messages from before this functionality was introduced.

477 SaveDate *time.Time `bstore:"default now"`

478

479 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

480 // masked IPs are used to classify incoming messages. They are left empty for

481 // messages matching a ruleset for forwarded messages.

482 RemoteIP string

483 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

484 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

485 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

486

487 // Only set if present and not an IP address. Unicode string. Empty for forwarded

488 // messages.

489 EHLODomain string `bstore:"index EHLODomain+Received"`

490 MailFrom string // With localpart and domain. Can be empty.

491 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

492 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

493 // messages, but see OrigMailFromDomain.

494 MailFromDomain string `bstore:"index MailFromDomain+Received"`

495 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

496 RcptToDomain string // Unicode string.

497

498 // Parsed "From" message header, used for reputation along with domain validation.

499 MsgFromLocalpart smtp.Localpart

500 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

501 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

502

503 // Simplified statements of the Validation fields below, used for incoming messages

504 // to check reputation.

505 EHLOValidated bool

506 MailFromValidated bool

507 MsgFromValidated bool

508

509 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

510 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

511 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

512

513 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

514 // DKIM domain that matched a ruleset's verified domain is left out, but included

515 // in OrigDKIMDomains.

516 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

517

518 // For forwarded messages,

519 OrigEHLODomain string

520 OrigDKIMDomains []string

521

522 // Canonicalized Message-Id, always lower-case and normalized quoting, without

523 // <>'s. Empty if missing. Used for matching message threads, and to prevent

524 // duplicate reject delivery.

525 MessageID string `bstore:"index"`

526 // lower-case: ../rfc/5256:495

527

528 // For matching threads in case there is no References/In-Reply-To header. It is

529 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

530 SubjectBase string `bstore:"index"`

531 // ../rfc/5256:90

532

533 // Hash of message. For rejects delivery in case there is no Message-ID, only set

534 // when delivered as reject.

535 MessageHash []byte

536

537 // ID of message starting this thread.

538 ThreadID int64 `bstore:"index"`

539 // IDs of parent messages, from closest parent to the root message. Parent messages

540 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

541 // never contain the message id itself (a cycle), and parent messages must

542 // reference the same ancestors. Moving a message to another mailbox keeps the

543 // message ID and changes the MailboxID (and UID) of the message, leaving threading

544 // parent ids intact.

545 ThreadParentIDs []int64

546 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

547 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

548 // have been deleted), or subject-based matching was done.

549 ThreadMissingLink bool

550 // If set, newly delivered child messages are automatically marked as read. This

551 // field is copied to new child messages. Changes are propagated to the webmail

552 // client.

553 ThreadMuted bool

554 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

555 // "on" (mode "unread" ignores it). This field is copied to new child message.

556 // Changes are propagated to the webmail client.

557 ThreadCollapsed bool

558

559 // If received message was known to match a mailing list rule (with modified junk

560 // filtering).

561 IsMailingList bool

562

563 // If this message is a DSN, generated by us or received. For DSNs, we don't look

564 // at the subject when matching threads.

565 DSN bool

566

567 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

568 ReceivedTLSCipherSuite uint16

569 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

570

571 Flags

572 // For keywords other than system flags or the basic well-known $-flags. Only in

573 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

574 // (for JMAP), sorted.

575 Keywords []string `bstore:"index"`

576 Size int64

577 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

578 MsgPrefix []byte // Typically holds received headers and/or header separator.

579

580 // If non-nil, a preview of the message based on text and/or html parts of the

581 // message. Used in the webmail and IMAP PREVIEW extension. If non-nil, it is empty

582 // if no preview could be created, or the message has not textual content or

583 // couldn't be parsed.

584 // Previews are typically created when delivering a message, but not when importing

585 // messages, for speed. Previews are generated on first request (in the webmail, or

586 // through the IMAP fetch attribute "PREVIEW" (without "LAZY")), and stored with

587 // the message at that time.

588 // The preview is at most 256 characters (can be more bytes), with detected quoted

589 // text replaced with "[...]". Previews typically end with a newline, callers may

590 // want to strip whitespace.

591 Preview *string

592

593 // ParsedBuf message structure. Currently saved as JSON of message.Part because

594 // bstore wasn't able to store recursive types when this was implemented. Created

595 // when first needed, and saved in the database.

596 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

597 ParsedBuf []byte

598}

599

600// MailboxCounts returns the delta to counts this message means for its

601// mailbox.

602func (m Message) MailboxCounts() (mc MailboxCounts) {

603 if m.Expunged {

604 return

605 }

606 if m.Deleted {

607 mc.Deleted++

608 } else {

609 mc.Total++

610 }

611 if !m.Seen {

612 mc.Unseen++

613 if !m.Deleted {

614 mc.Unread++

615 }

616 }

617 mc.Size += m.Size

618 return

619}

620

621func (m Message) ChangeAddUID(mb Mailbox) ChangeAddUID {

622 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords, mb.MessageCountIMAP(), uint32(mb.MailboxCounts.Unseen)}

623}

624

625func (m Message) ChangeFlags(orig Flags, mb Mailbox) ChangeFlags {

626 mask := m.Flags.Changed(orig)

627 return ChangeFlags{m.MailboxID, m.UID, m.ModSeq, mask, m.Flags, m.Keywords, mb.UIDValidity, uint32(mb.MailboxCounts.Unseen)}

628}

629

630func (m Message) ChangeThread() ChangeThread {

631 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

632}

633

634// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

635// database is sent to the client as 1, because modseq 0 is special in IMAP.

636// ModSeq coming from the client are of type int64.

637type ModSeq int64

638

639func (ms ModSeq) Client() int64 {

640 if ms == 0 {

641 return 1

642 }

643 return int64(ms)

644}

645

646// ModSeqFromClient converts a modseq from a client to a modseq for internal

647// use, e.g. in a database query.

648// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

649func ModSeqFromClient(modseq int64) ModSeq {

650 if modseq == 1 {

651 return 0

652 }

653 return ModSeq(modseq)

654}

655

656// Erase clears fields from a Message that are no longer needed after actually

657// removing the message file from the file system, after all references to the

658// message have gone away. Only the fields necessary for synchronisation are kept.

659func (m *Message) erase() {

660 if !m.Expunged {

661 panic("erase called on non-expunged message")

662 }

663 *m = Message{

664 ID: m.ID,

665 UID: m.UID,

666 MailboxID: m.MailboxID,

667 CreateSeq: m.CreateSeq,

668 ModSeq: m.ModSeq,

669 Expunged: true,

670 ThreadID: m.ThreadID,

671 }

672}

673

674// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based

675// on the part.

676func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

677 m.DSN = part.IsDSN()

678

679 if part.Envelope == nil {

680 return

681 }

682 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

683 if err != nil {

684 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

685 } else if raw {

686 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

687 }

688 m.MessageID = messageID

689 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

690}

691

692// LoadPart returns a message.Part by reading from m.ParsedBuf.

693func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

694 if m.ParsedBuf == nil {

695 return message.Part{}, fmt.Errorf("message not parsed")

696 }

697 var p message.Part

698 err := json.Unmarshal(m.ParsedBuf, &p)

699 if err != nil {

700 return p, fmt.Errorf("unmarshal message part")

701 }

702 p.SetReaderAt(r)

703 return p, nil

704}

705

706// NeedsTraining returns whether message needs a training update, based on

707// TrainedJunk (current training status) and new Junk/Notjunk flags.

708func (m Message) NeedsTraining() bool {

709 needs, _, _, _, _ := m.needsTraining()

710 return needs

711}

712

713func (m Message) needsTraining() (needs, untrain, untrainJunk, train, trainJunk bool) {

714 untrain = m.TrainedJunk != nil

715 untrainJunk = untrain && *m.TrainedJunk

716 train = m.Junk != m.Notjunk

717 trainJunk = m.Junk

718 needs = untrain != train || untrain && train && untrainJunk != trainJunk

719 return

720}

721

722// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

723// used when delivering/moving/copying messages to a mailbox. Mail clients are not

724// very helpful with setting junk/notjunk flags. But clients can move/copy messages

725// to other mailboxes. So we set flags when clients move a message.

726func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

727 if mb.Junk {

728 m.Junk = true

729 m.Notjunk = false

730 return

731 }

732

733 if !conf.AutomaticJunkFlags.Enabled {

734 return

735 }

736

737 lmailbox := strings.ToLower(mb.Name)

738

739 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

740 m.Junk = true

741 m.Notjunk = false

742 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

743 m.Junk = false

744 m.Notjunk = false

745 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

746 m.Junk = false

747 m.Notjunk = true

748 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

749 m.Junk = true

750 m.Notjunk = false

751 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

752 m.Junk = false

753 m.Notjunk = false

754 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

755 m.Junk = false

756 m.Notjunk = true

757 }

758}

759

760// JunkFlagsForMailboxMove sets Junk and Notjunk flags for a mailbox move. A

761// message moved or copied from Introbox to its originally intended mailbox or

762// another mailbox that marks it as non-junk, it is marked as a positive

763// interaction and used for reputation during future deliveries. A move or copy to

764// a junk mailbox records a negative interaction for the intended mailbox.

765func (m *Message) JunkFlagsForMailboxMove(mbSrc, mbDst Mailbox, conf config.Account) {

766 m.JunkFlagsForMailbox(mbDst, conf)

767 if mbSrc.Name != conf.Introbox || m.MailboxDestinedID == 0 {

768 return

769 }

770 if m.MailboxDestinedID == mbDst.ID && !m.Junk && !m.Notjunk {

771 m.Junk = false

772 m.Notjunk = true

773 }

774 if m.Notjunk || m.Junk || m.MailboxDestinedID == mbDst.ID {

775 m.MailboxOrigID = m.MailboxDestinedID

776 m.MailboxDestinedID = 0

777 }

778}

779

780// Recipient represents the recipient of a message. It is tracked to allow

781// first-time incoming replies from users this account has sent messages to. When a

782// mailbox is added to the Sent mailbox the message is parsed and recipients are

783// inserted as recipient. Recipients are never removed other than for removing the

784// message. On move/copy of a message, recipients aren't modified either. For IMAP,

785// this assumes a client simply appends messages to the Sent mailbox (as opposed to

786// copying messages from some place).

787type Recipient struct {

788 ID int64

789 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

790 Localpart string `bstore:"nonzero"` // Encoded localpart.

791 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

792 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

793 Sent time.Time `bstore:"nonzero"`

794}

795

796// Outgoing is a message submitted for delivery from the queue. Used to enforce

797// maximum outgoing messages.

798type Outgoing struct {

799 ID int64

800 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

801 Submitted time.Time `bstore:"nonzero,default now"`

802}

803

804// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

805// during most recent connection (delivery attempt).

806type RecipientDomainTLS struct {

807 Domain string // Unicode.

808 Updated time.Time `bstore:"default now"`

809 STARTTLS bool // Supports STARTTLS.

810 RequireTLS bool // Supports RequireTLS SMTP extension.

811}

812

813// DiskUsage tracks quota use.

814type DiskUsage struct {

815 ID int64 // Always one record with ID 1.

816 MessageSize int64 // Sum of all messages, for quota accounting.

817}

818

819// SessionToken and CSRFToken are types to prevent mixing them up.

820// Base64 raw url encoded.

821type SessionToken string

822type CSRFToken string

823

824// LoginSession represents a login session. We keep a limited number of sessions

825// for a user, removing the oldest session when a new one is created.

826type LoginSession struct {

827 ID int64

828 Created time.Time `bstore:"nonzero,default now"` // Of original login.

829 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

830 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

831 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

832 AccountName string `bstore:"nonzero"`

833 LoginAddress string `bstore:"nonzero"`

834

835 // Set when loading from database.

836 sessionToken SessionToken

837 csrfToken CSRFToken

838}

839

840// Quoting is a setting for how to quote in replies/forwards.

841type Quoting string

842

843const (

844 Default Quoting = "" // Bottom-quote if text is selected, top-quote otherwise.

845 Bottom Quoting = "bottom"

846 Top Quoting = "top"

847)

848

849// Settings are webmail client settings.

850type Settings struct {

851 ID uint8 // Singleton ID 1.

852

853 Signature string

854 Quoting Quoting

855

856 // Whether to show the bars underneath the address input fields indicating

857 // starttls/dnssec/dane/mtasts/requiretls support by address.

858 ShowAddressSecurity bool

859

860 // Show HTML version of message by default, instead of plain text.

861 ShowHTML bool

862

863 // If true, don't show shortcuts in webmail after mouse interaction.

864 NoShowShortcuts bool

865

866 // Additional headers to display in message view. E.g. Delivered-To, User-Agent, X-Mox-Reason.

867 ShowHeaders []string

868}

869

870// ViewMode how a message should be viewed: its text parts, html parts, or html

871// with loading external resources.

872type ViewMode string

873

874const (

875 ModeText ViewMode = "text"

876 ModeHTML ViewMode = "html"

877 ModeHTMLExt ViewMode = "htmlext" // HTML with external resources.

878)

879

880// FromAddressSettings are webmail client settings per "From" address.

881type FromAddressSettings struct {

882 FromAddress string // Unicode.

883 ViewMode ViewMode

884}

885

886// RulesetNoListID records a user "no" response to the question of

887// creating/removing a ruleset after moving a message with list-id header from/to

888// the inbox.

889type RulesetNoListID struct {

890 ID int64

891 RcptToAddress string `bstore:"nonzero"`

892 ListID string `bstore:"nonzero"`

893 ToInbox bool // Otherwise from Inbox to other mailbox.

894}

895

896// RulesetNoMsgFrom records a user "no" response to the question of

897// creating/moveing a ruleset after moving a mesage with message "from" address

898// from/to the inbox.

899type RulesetNoMsgFrom struct {

900 ID int64

901 RcptToAddress string `bstore:"nonzero"`

902 MsgFromAddress string `bstore:"nonzero"` // Unicode.

903 ToInbox bool // Otherwise from Inbox to other mailbox.

904}

905

906// RulesetNoMailbox represents a "never from/to this mailbox" response to the

907// question of adding/removing a ruleset after moving a message.

908type RulesetNoMailbox struct {

909 ID int64

910

911 // The mailbox from/to which the move has happened.

912 // Not a references, if mailbox is deleted, an entry becomes ineffective.

913 MailboxID int64 `bstore:"nonzero"`

914 ToMailbox bool // Whether MailboxID is the destination of the move (instead of source).

915}

916

917// MessageErase represents the need to remove a message file from disk, and clear

918// message fields from the database, but only when the last reference to the

919// message is gone (all IMAP sessions need to have applied the changes indicating

920// message removal).

921type MessageErase struct {

922 ID int64 // Same ID as Message.ID.

923

924 // Whether to subtract the size from the total disk usage. Useful for moving

925 // messages, which involves duplicating the message temporarily, while there are

926 // still references in the old mailbox, but which isn't counted as using twice the

927 // disk space..

928 SkipUpdateDiskUsage bool

929}

930

931// Types stored in DB.

932var DBTypes = []any{

933 NextUIDValidity{},

934 Message{},

935 Recipient{},

936 Mailbox{},

937 Subscription{},

938 Outgoing{},

939 Password{},

940 Subjectpass{},

941 SyncState{},

942 Upgrade{},

943 RecipientDomainTLS{},

944 DiskUsage{},

945 LoginSession{},

946 Settings{},

947 FromAddressSettings{},

948 RulesetNoListID{},

949 RulesetNoMsgFrom{},

950 RulesetNoMailbox{},

951 Annotation{},

952 MessageErase{},

953}

954

955// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

956type Account struct {

957 Name string // Name, according to configuration.

958 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

959 DBPath string // Path to database with mailboxes, messages, etc.

960 DB *bstore.DB // Open database connection.

961

962 // Channel that is closed if/when account has/gets "threads" accounting (see

963 // Upgrade.Threads).

964 threadsCompleted chan struct{}

965 // If threads upgrade completed with error, this is set. Used for warning during

966 // delivery, or aborting when importing.

967 threadsErr error

968

969 // Message directory of last delivery. Used to check we don't have to make that

970 // directory when delivering.

971 lastMsgDir string

972

973 // If set, consistency checks won't fail on message ModSeq/CreateSeq being zero.

974 skipMessageZeroSeqCheck bool

975

976 // Write lock must be held when modifying account/mailbox/message/flags/annotations

977 // if the change needs to be synchronized with client connections by broadcasting

978 // the changes. Changes that are not protocol-visible do not require a lock, the

979 // database transactions isolate activity, though locking may be necessary to

980 // protect in-memory-only access.

981 //

982 // Read lock for reading mailboxes/messages as a consistent snapsnot (i.e. not

983 // concurrent changes). For longer transactions, e.g. when reading many messages,

984 // the lock can be released while continuing to read from the transaction.

985 //

986 // When making changes to mailboxes/messages, changes must be broadcasted before

987 // releasing the lock to ensure proper UID ordering.

988 sync.RWMutex

989

990 // Reference count, while >0, this account is alive and shared. Protected by

991 // openAccounts, not by account wlock.

992 nused int

993 removed bool // Marked for removal. Last close removes the account directory.

994 closed chan struct{} // Closed when last reference is gone.

995}

996

997type Upgrade struct {

998 ID byte

999 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

1000 MailboxModSeq bool // Whether mailboxes have been assigned modseqs.

1001 MailboxParentID bool // Setting ParentID on mailboxes.

1002 MailboxCounts bool // Global flag about whether we have mailbox flags. Instead of previous per-mailbox boolean.

1003 MessageParseVersion int // If different than latest, all messages will be reparsed.

1004}

1005

1006const MessageParseVersionLatest = 2

1007

1008// upgradeInit is the value for new account database, which don't need any upgrading.

1009var upgradeInit = Upgrade{

1010 ID: 1, // Singleton.

1011 Threads: 2,

1012 MailboxModSeq: true,

1013 MailboxParentID: true,

1014 MailboxCounts: true,

1015 MessageParseVersion: MessageParseVersionLatest,

1016}

1017

1018// InitialUIDValidity returns a UIDValidity used for initializing an account.

1019// It can be replaced during tests with a predictable value.

1020var InitialUIDValidity = func() uint32 {

1021 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

1022}

1023

1024var openAccounts = struct {

1025 sync.Mutex

1026 names map[string]*Account

1027}{

1028 names: map[string]*Account{},

1029}

1030

1031func closeAccount(acc *Account) (rerr error) {

1032 // If we need to remove the account files, we do so without the accounts lock.

1033 remove := false

1034 defer func() {

1035 if remove {

1036 log := mlog.New("store", nil)

1037 err := removeAccount(log, acc.Name)

1038 if rerr == nil {

1039 rerr = err

1040 }

1041 close(acc.closed)

1042 }

1043 }()

1044

1045 openAccounts.Lock()

1046 defer openAccounts.Unlock()

1047 acc.nused--

1048 if acc.nused > 0 {

1049 return

1050 }

1051 remove = acc.removed

1052

1053 defer func() {

1054 err := acc.DB.Close()

1055 acc.DB = nil

1056 delete(openAccounts.names, acc.Name)

1057 if !remove {

1058 close(acc.closed)

1059 }

1060

1061 if rerr == nil {

1062 rerr = err

1063 }

1064 }()

1065

1066 // Verify there are no more pending MessageErase records.

1067 l, err := bstore.QueryDB[MessageErase](context.TODO(), acc.DB).List()

1068 if err != nil {

1069 return fmt.Errorf("listing messageerase records: %v", err)

1070 } else if len(l) > 0 {

1071 return fmt.Errorf("messageerase records still present after last account reference is gone: %v", l)

1072 }

1073

1074 return nil

1075}

1076

1077// removeAccount moves the account directory for an account away and removes

1078// all files, and removes the AccountRemove struct from the database.

1079func removeAccount(log mlog.Log, accountName string) error {

1080 log = log.With(slog.String("account", accountName))

1081 log.Info("removing account directory and files")

1082

1083 // First move the account directory away.

1084 odir := filepath.Join(mox.DataDirPath("accounts"), accountName)

1085 tmpdir := filepath.Join(mox.DataDirPath("tmp"), "oldaccount-"+accountName)

1086 if err := os.Rename(odir, tmpdir); err != nil {

1087 return fmt.Errorf("moving account data directory %q out of the way to %q (account not removed): %v", odir, tmpdir, err)

1088 }

1089

1090 var errs []error

1091

1092 // Commit removal to database.

1093 err := AuthDB.Write(context.Background(), func(tx *bstore.Tx) error {

1094 if err := tx.Delete(&AccountRemove{accountName}); err != nil {

1095 return fmt.Errorf("deleting account removal request: %v", err)

1096 }

1097 if err := tlsPublicKeyRemoveForAccount(tx, accountName); err != nil {

1098 return fmt.Errorf("removing tls public keys for account: %v", err)

1099 }

1100

1101 if err := loginAttemptRemoveAccount(tx, accountName); err != nil {

1102 return fmt.Errorf("removing historic login attempts for account: %v", err)

1103 }

1104 return nil

1105 })

1106 if err != nil {

1107 errs = append(errs, fmt.Errorf("remove account from database: %w", err))

1108 }

1109

1110 // Remove the account directory and its message and other files.

1111 if err := os.RemoveAll(tmpdir); err != nil {

1112 errs = append(errs, fmt.Errorf("removing account data directory %q that was moved to %q: %v", odir, tmpdir, err))

1113 }

1114

1115 return errors.Join(errs...)

1116}

1117

1118// OpenAccount opens an account by name.

1119//

1120// No additional data path prefix or ".db" suffix should be added to the name.

1121// A single shared account exists per name.

1122func OpenAccount(log mlog.Log, name string, checkLoginDisabled bool) (*Account, error) {

1123 openAccounts.Lock()

1124 defer openAccounts.Unlock()

1125 if acc, ok := openAccounts.names[name]; ok {

1126 if acc.removed {

1127 return nil, fmt.Errorf("account has been removed")

1128 }

1129

1130 acc.nused++

1131 return acc, nil

1132 }

1133

1134 if a, ok := mox.Conf.Account(name); !ok {

1135 return nil, ErrAccountUnknown

1136 } else if checkLoginDisabled && a.LoginDisabled != "" {

1137 return nil, fmt.Errorf("%w: %s", ErrLoginDisabled, a.LoginDisabled)

1138 }

1139

1140 acc, err := openAccount(log, name)

1141 if err != nil {

1142 return nil, err

1143 }

1144 openAccounts.names[name] = acc

1145 return acc, nil

1146}

1147

1148// openAccount opens an existing account, or creates it if it is missing.

1149// Called with openAccounts lock held.

1150func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

1151 dir := filepath.Join(mox.DataDirPath("accounts"), name)

1152 return OpenAccountDB(log, dir, name)

1153}

1154

1155// OpenAccountDB opens an account database file and returns an initialized account

1156// or error. Only exported for use by subcommands that verify the database file.

1157// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

1158func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

1159 log = log.With(slog.String("account", accountName))

1160

1161 dbpath := filepath.Join(accountDir, "index.db")

1162

1163 // Create account if it doesn't exist yet.

1164 isNew := false

1165 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

1166 isNew = true

1167 os.MkdirAll(accountDir, 0770)

1168 }

1169

1170 opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: moxvar.RegisterLogger(dbpath, log.Logger)}

1171 db, err := bstore.Open(context.TODO(), dbpath, &opts, DBTypes...)

1172 if err != nil {

1173 return nil, err

1174 }

1175

1176 defer func() {

1177 if rerr != nil {

1178 err := db.Close()

1179 log.Check(err, "closing database file after error")

1180 if isNew {

1181 err := os.Remove(dbpath)

1182 log.Check(err, "removing new database file after error")

1183 }

1184 }

1185 }()

1186

1187 acc := &Account{

1188 Name: accountName,

1189 Dir: accountDir,

1190 DBPath: dbpath,

1191 DB: db,

1192 nused: 1,

1193 closed: make(chan struct{}),

1194 threadsCompleted: make(chan struct{}),

1195 }

1196

1197 if isNew {

1198 if err := initAccount(db); err != nil {

1199 return nil, fmt.Errorf("initializing account: %v", err)

1200 }

1201

1202 close(acc.threadsCompleted)

1203 return acc, nil

1204 }

1205

1206 // Ensure singletons are present, like DiskUsage and Settings.

1207 // Process pending MessageErase records. Check that next the message ID assigned by

1208 // the database does not already have a file on disk, or increase the sequence so

1209 // it doesn't.

1210 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

1211 if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent {

1212 if err := tx.Insert(&Settings{ID: 1, ShowAddressSecurity: true}); err != nil {

1213 return err

1214 }

1215 }

1216

1217 du := DiskUsage{ID: 1}

1218 err = tx.Get(&du)

1219 if err == bstore.ErrAbsent {

1220 // No DiskUsage record yet, calculate total size and insert.

1221 err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb Mailbox) error {

1222 du.MessageSize += mb.Size

1223 return nil

1224 })

1225 if err != nil {

1226 return err

1227 }

1228 if err := tx.Insert(&du); err != nil {

1229 return err

1230 }

1231 } else if err != nil {

1232 return err

1233 }

1234

1235 var erase []MessageErase

1236 if _, err := bstore.QueryTx[MessageErase](tx).Gather(&erase).Delete(); err != nil {

1237 return fmt.Errorf("fetching messages to erase: %w", err)

1238 }

1239 if len(erase) > 0 {

1240 log.Debug("deleting message files from message erase records", slog.Int("count", len(erase)))

1241 }

1242 var duChanged bool

1243 for _, me := range erase {

1244 // Clear the fields from the message not needed for synchronization.

1245 m := Message{ID: me.ID}

1246 if err := tx.Get(&m); err != nil {

1247 return fmt.Errorf("get message %d to expunge: %w", me.ID, err)

1248 } else if !m.Expunged {

1249 return fmt.Errorf("message %d to erase is not expunged", m.ID)

1250 }

1251

1252 // We remove before we update/commit the database, so we are sure we don't leave

1253 // files behind in case of an error/crash.

1254 p := acc.MessagePath(me.ID)

1255 err := os.Remove(p)

1256 log.Check(err, "removing message file for expunged message", slog.String("path", p))

1257

1258 if !me.SkipUpdateDiskUsage {

1259 du.MessageSize -= m.Size

1260 duChanged = true

1261 }

1262

1263 m.erase()

1264 if err := tx.Update(&m); err != nil {

1265 return fmt.Errorf("save erase of message %d in database: %w", m.ID, err)

1266 }

1267 }

1268

1269 if duChanged {

1270 if err := tx.Update(&du); err != nil {

1271 return fmt.Errorf("saving disk usage after erasing messages: %w", err)

1272 }

1273 }

1274

1275 // Ensure the message directories don't have a higher message ID than occurs in our

1276 // database. If so, increase the next ID used for inserting a message to prevent

1277 // clash during delivery.

1278 last, err := bstore.QueryTx[Message](tx).SortDesc("ID").Limit(1).Get()

1279 if err != nil && err != bstore.ErrAbsent {

1280 return fmt.Errorf("querying last message: %v", err)

1281 }

1282

1283 // We look in the directory where the message is stored (the id can be 0, which is fine).

1284 maxDBID := last.ID

1285 p := acc.MessagePath(maxDBID)

1286 dir := filepath.Dir(p)

1287 maxFSID := maxDBID

1288 // We also try looking for the next directories that would be created for messages,

1289 // until one doesn't exist anymore. We never delete these directories.

1290 for {

1291 np := acc.MessagePath(maxFSID + msgFilesPerDir)

1292 ndir := filepath.Dir(np)

1293 if _, err := os.Stat(ndir); err == nil {

1294 maxFSID = (maxFSID + msgFilesPerDir) &^ (msgFilesPerDir - 1) // First ID for dir.

1295 dir = ndir

1296 } else if errors.Is(err, fs.ErrNotExist) {

1297 break

1298 } else {

1299 return fmt.Errorf("stat next message directory %q: %v", ndir, err)

1300 }

1301 }

1302 // Find highest numbered file within the directory.

1303 entries, err := os.ReadDir(dir)

1304 if err != nil && !errors.Is(err, fs.ErrNotExist) {

1305 return fmt.Errorf("read message directory %q: %v", dir, err)

1306 }

1307 dirFirstID := maxFSID &^ (msgFilesPerDir - 1)

1308 for _, e := range entries {

1309 id, err := strconv.ParseInt(e.Name(), 10, 64)

1310 if err == nil && (id < dirFirstID || id >= dirFirstID+msgFilesPerDir) {

1311 err = fmt.Errorf("directory %s has message id %d outside of range [%d - %d), ignoring", dir, id, dirFirstID, dirFirstID+msgFilesPerDir)

1312 }

1313 if err != nil {

1314 p := filepath.Join(dir, e.Name())

1315 log.Errorx("unrecognized file in message directory, parsing filename as number", err, slog.String("path", p))

1316 } else {

1317 maxFSID = max(maxFSID, id)

1318 }

1319 }

1320 // Warn if we need to increase the message ID in the database.

1321 var mailboxID int64

1322 if maxFSID > maxDBID {

1323 log.Warn("unexpected message file with higher message id than highest id in database, moving database id sequence forward to prevent clashes during future deliveries", slog.Int64("maxdbmsgid", maxDBID), slog.Int64("maxfilemsgid", maxFSID))

1324

1325 mb, err := bstore.QueryTx[Mailbox](tx).Limit(1).Get()

1326 if err != nil {

1327 return fmt.Errorf("get a mailbox: %v", err)

1328 }

1329 mailboxID = mb.ID

1330 }

1331 for maxFSID > maxDBID {

1332 // Set fields that must be non-zero.

1333 m := Message{

1334 UID: ^UID(0),

1335 MailboxID: mailboxID,

1336 }

1337 // Insert and delete to increase the sequence, silly but effective.

1338 if err := tx.Insert(&m); err != nil {

1339 return fmt.Errorf("inserting message to increase id: %v", err)

1340 }

1341 if err := tx.Delete(&m); err != nil {

1342 return fmt.Errorf("deleting message after increasing id: %v", err)

1343 }

1344 maxDBID = m.ID

1345 }

1346

1347 return nil

1348 })

1349 if err != nil {

1350 return nil, fmt.Errorf("calculating counts for mailbox, inserting settings, expunging messages: %v", err)

1351 }

1352

1353 up := Upgrade{ID: 1}

1354 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

1355 err := tx.Get(&up)

1356 if err == bstore.ErrAbsent {

1357 if err := tx.Insert(&up); err != nil {

1358 return fmt.Errorf("inserting initial upgrade record: %v", err)

1359 }

1360 err = nil

1361 }

1362 return err

1363 })

1364 if err != nil {

1365 return nil, fmt.Errorf("checking message threading: %v", err)

1366 }

1367

1368 // Ensure all mailboxes have a modseq based on highest modseq message in each

1369 // mailbox, and a createseq.

1370 if !up.MailboxModSeq {

1371 log.Debug("upgrade: adding modseq to each mailbox")

1372 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1373 var modseq ModSeq

1374

1375 mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).List()

1376 if err != nil {

1377 return fmt.Errorf("listing mailboxes: %v", err)

1378 }

1379 for _, mb := range mbl {

1380 // Get current highest modseq of message in account.

1381 qms := bstore.QueryTx[Message](tx)

1382 qms.FilterNonzero(Message{MailboxID: mb.ID})

1383 qms.SortDesc("ModSeq")

1384 qms.Limit(1)

1385 m, err := qms.Get()

1386 if err == nil {

1387 mb.ModSeq = ModSeq(m.ModSeq.Client())

1388 } else if err == bstore.ErrAbsent {

1389 if modseq == 0 {

1390 modseq, err = acc.NextModSeq(tx)

1391 if err != nil {

1392 return fmt.Errorf("get next mod seq for mailbox without messages: %v", err)

1393 }

1394 }

1395 mb.ModSeq = modseq

1396 } else {

1397 return fmt.Errorf("looking up highest modseq for mailbox: %v", err)

1398 }

1399 mb.CreateSeq = 1

1400 if err := tx.Update(&mb); err != nil {

1401 return fmt.Errorf("updating mailbox with modseq: %v", err)

1402 }

1403 }

1404

1405 up.MailboxModSeq = true

1406 if err := tx.Update(&up); err != nil {

1407 return fmt.Errorf("marking upgrade done: %v", err)

1408 }

1409

1410 return nil

1411 })

1412 if err != nil {

1413 return nil, fmt.Errorf("upgrade: adding modseq to each mailbox: %v", err)

1414 }

1415 }

1416

1417 // Add ParentID to mailboxes.

1418 if !up.MailboxParentID {

1419 log.Debug("upgrade: setting parentid on each mailbox")

1420

1421 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1422 mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).SortAsc("Name").List()

1423 if err != nil {

1424 return fmt.Errorf("listing mailboxes: %w", err)

1425 }

1426

1427 names := map[string]Mailbox{}

1428 for _, mb := range mbl {

1429 names[mb.Name] = mb

1430 }

1431

1432 var modseq ModSeq

1433

1434 // Ensure a parent mailbox for name exists, creating it if needed, including any

1435 // grandparents, up to the top.

1436 var ensureParentMailboxID func(name string) (int64, error)

1437 ensureParentMailboxID = func(name string) (int64, error) {

1438 parentName := mox.ParentMailboxName(name)

1439 if parentName == "" {

1440 return 0, nil

1441 }

1442 parent := names[parentName]

1443 if parent.ID != 0 {

1444 return parent.ID, nil

1445 }

1446

1447 parentParentID, err := ensureParentMailboxID(parentName)

1448 if err != nil {

1449 return 0, fmt.Errorf("creating parent mailbox %q: %w", parentName, err)

1450 }

1451

1452 if modseq == 0 {

1453 modseq, err = a.NextModSeq(tx)

1454 if err != nil {

1455 return 0, fmt.Errorf("get next modseq: %w", err)

1456 }

1457 }

1458

1459 uidvalidity, err := a.NextUIDValidity(tx)

1460 if err != nil {

1461 return 0, fmt.Errorf("next uid validity: %w", err)

1462 }

1463

1464 parent = Mailbox{

1465 CreateSeq: modseq,

1466 ModSeq: modseq,

1467 ParentID: parentParentID,

1468 Name: parentName,

1469 UIDValidity: uidvalidity,

1470 UIDNext: 1,

1471 SpecialUse: SpecialUse{},

1472 HaveCounts: true,

1473 }

1474 if err := tx.Insert(&parent); err != nil {

1475 return 0, fmt.Errorf("creating parent mailbox: %w", err)

1476 }

1477 return parent.ID, nil

1478 }

1479

1480 for _, mb := range mbl {

1481 parentID, err := ensureParentMailboxID(mb.Name)

1482 if err != nil {

1483 return fmt.Errorf("creating missing parent mailbox for mailbox %q: %w", mb.Name, err)

1484 }

1485 mb.ParentID = parentID

1486 if err := tx.Update(&mb); err != nil {

1487 return fmt.Errorf("update mailbox with parentid: %w", err)

1488 }

1489 }

1490

1491 up.MailboxParentID = true

1492 if err := tx.Update(&up); err != nil {

1493 return fmt.Errorf("marking upgrade done: %w", err)

1494 }

1495 return nil

1496 })

1497 if err != nil {

1498 return nil, fmt.Errorf("upgrade: setting parentid on each mailbox: %w", err)

1499 }

1500 }

1501

1502 if !up.MailboxCounts {

1503 log.Debug("upgrade: ensuring all mailboxes have message counts")

1504

1505 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1506 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

1507 mc, err := mb.CalculateCounts(tx)

1508 if err != nil {

1509 return err

1510 }

1511 mb.HaveCounts = true

1512 mb.MailboxCounts = mc

1513 return tx.Update(&mb)

1514 })

1515 if err != nil {

1516 return err

1517 }

1518

1519 up.MailboxCounts = true

1520 if err := tx.Update(&up); err != nil {

1521 return fmt.Errorf("marking upgrade done: %w", err)

1522 }

1523 return nil

1524 })

1525 if err != nil {

1526 return nil, fmt.Errorf("upgrade: ensuring message counts on all mailboxes")

1527 }

1528 }

1529

1530 if up.MessageParseVersion != MessageParseVersionLatest {

1531 log.Debug("upgrade: reparsing message for mime structures for new message parse version", slog.Int("current", up.MessageParseVersion), slog.Int("latest", MessageParseVersionLatest))

1532

1533 // Unless we also need to upgrade threading, we'll be reparsing messages in the

1534 // background so opening of the account is quick.

1535 done := make(chan error, 1)

1536 bg := up.Threads == 2

1537

1538 // Increase account use before holding on to account in background.

1539 // Caller holds the lock. The goroutine below decreases nused by calling

1540 // closeAccount.

1541 acc.nused++

1542

1543 go func() {

1544 start := time.Now()

1545

1546 var rerr error

1547 defer func() {

1548 x := recover()

1549 if x != nil {

1550 rerr = fmt.Errorf("unhandled panic: %v", x)

1551 log.Error("unhandled panic reparsing messages", slog.Any("err", x))

1552 debug.PrintStack()

1553 metrics.PanicInc(metrics.Store)

1554 }

1555

1556 if bg && rerr != nil {

1557 log.Errorx("upgrade failed: reparsing message for mime structures for new message parse version", rerr, slog.Duration("duration", time.Since(start)))

1558 }

1559 done <- rerr

1560

1561 // Must be done at end of defer. Our parent context/goroutine has openAccounts lock

1562 // held, so we won't make progress until after the enclosing method has returned.

1563 err := closeAccount(acc)

1564 log.Check(err, "closing account after reparsing messages")

1565 }()

1566

1567 var total int

1568 total, rerr = acc.ReparseMessages(mox.Shutdown, log)

1569 if rerr != nil {

1570 rerr = fmt.Errorf("reparsing messages and updating mime structures in message index: %w", rerr)

1571 return

1572 }

1573

1574 up.MessageParseVersion = MessageParseVersionLatest

1575 rerr = acc.DB.Update(context.TODO(), &up)

1576 if rerr != nil {

1577 rerr = fmt.Errorf("marking latest message parse version: %w", rerr)

1578 return

1579 }

1580

1581 log.Info("upgrade completed: reparsing message for mime structures for new message parse version", slog.Int("total", total), slog.Duration("duration", time.Since(start)))

1582 }()

1583

1584 if !bg {

1585 err := <-done

1586 if err != nil {

1587 return nil, err

1588 }

1589 }

1590 }

1591

1592 if up.Threads == 2 {

1593 close(acc.threadsCompleted)

1594 return acc, nil

1595 }

1596

1597 // Increase account use before holding on to account in background.

1598 // Caller holds the lock. The goroutine below decreases nused by calling

1599 // closeAccount.

1600 acc.nused++

1601

1602 // Ensure all messages have a MessageID and SubjectBase, which are needed when

1603 // matching threads.

1604 // Then assign messages to threads, in the same way we do during imports.

1605 log.Info("upgrading account for threading, in background")

1606 go func() {

1607 defer func() {

1608 err := closeAccount(acc)

1609 log.Check(err, "closing use of account after upgrading account storage for threads")

1610

1611 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

1612 close(acc.threadsCompleted)

1613 }()

1614

1615 defer func() {

1616 x := recover() // Should not happen, but don't take program down if it does.

1617 if x != nil {

1618 log.Error("upgradeThreads panic", slog.Any("err", x))

1619 debug.PrintStack()

1620 metrics.PanicInc(metrics.Upgradethreads)

1621 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

1622 }

1623 }()

1624

1625 err := upgradeThreads(mox.Shutdown, log, acc, up)

1626 if err != nil {

1627 a.threadsErr = err

1628 log.Errorx("upgrading account for threading, aborted", err)

1629 } else {

1630 log.Info("upgrading account for threading, completed")

1631 }

1632 }()

1633 return acc, nil

1634}

1635

1636// ThreadingWait blocks until the one-time account threading upgrade for the

1637// account has completed, and returns an error if not successful.

1638//

1639// To be used before starting an import of messages.

1640func (a *Account) ThreadingWait(log mlog.Log) error {

1641 select {

1642 case <-a.threadsCompleted:

1643 return a.threadsErr

1644 default:

1645 }

1646 log.Debug("waiting for account upgrade to complete")

1647

1648 <-a.threadsCompleted

1649 return a.threadsErr

1650}

1651

1652func initAccount(db *bstore.DB) error {

1653 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

1654 uidvalidity := InitialUIDValidity()

1655

1656 if err := tx.Insert(&upgradeInit); err != nil {

1657 return err

1658 }

1659 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

1660 return err

1661 }

1662 if err := tx.Insert(&Settings{ID: 1}); err != nil {

1663 return err

1664 }

1665

1666 modseq, err := nextModSeq(tx)

1667 if err != nil {

1668 return fmt.Errorf("get next modseq: %v", err)

1669 }

1670

1671 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

1672 // Deprecated in favor of InitialMailboxes.

1673 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

1674 mailboxes := []string{"Inbox"}

1675 for _, name := range defaultMailboxes {

1676 if strings.EqualFold(name, "Inbox") {

1677 continue

1678 }

1679 mailboxes = append(mailboxes, name)

1680 }

1681 for _, name := range mailboxes {

1682 mb := Mailbox{

1683 CreateSeq: modseq,

1684 ModSeq: modseq,

1685 ParentID: 0,

1686 Name: name,

1687 UIDValidity: uidvalidity,

1688 UIDNext: 1,

1689 HaveCounts: true,

1690 }

1691 if strings.HasPrefix(name, "Archive") {

1692 mb.Archive = true

1693 } else if strings.HasPrefix(name, "Drafts") {

1694 mb.Draft = true

1695 } else if strings.HasPrefix(name, "Junk") {

1696 mb.Junk = true

1697 } else if strings.HasPrefix(name, "Sent") {

1698 mb.Sent = true

1699 } else if strings.HasPrefix(name, "Trash") {

1700 mb.Trash = true

1701 }

1702 if err := tx.Insert(&mb); err != nil {

1703 return fmt.Errorf("creating mailbox: %w", err)

1704 }

1705 if err := tx.Insert(&Subscription{name}); err != nil {

1706 return fmt.Errorf("adding subscription: %w", err)

1707 }

1708 }

1709 } else {

1710 mailboxes := mox.Conf.Static.InitialMailboxes

1711 var zerouse config.SpecialUseMailboxes

1712 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1713 mailboxes = DefaultInitialMailboxes

1714 }

1715

1716 add := func(name string, use SpecialUse) error {

1717 mb := Mailbox{

1718 CreateSeq: modseq,

1719 ModSeq: modseq,

1720 ParentID: 0,

1721 Name: name,

1722 UIDValidity: uidvalidity,

1723 UIDNext: 1,

1724 SpecialUse: use,

1725 HaveCounts: true,

1726 }

1727 if err := tx.Insert(&mb); err != nil {

1728 return fmt.Errorf("creating mailbox: %w", err)

1729 }

1730 if err := tx.Insert(&Subscription{name}); err != nil {

1731 return fmt.Errorf("adding subscription: %w", err)

1732 }

1733 return nil

1734 }

1735 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1736 if nameOpt == "" {

1737 return nil

1738 }

1739 return add(nameOpt, use)

1740 }

1741 l := []struct {

1742 nameOpt string

1743 use SpecialUse

1744 }{

1745 {"Inbox", SpecialUse{}},

1746 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1747 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1748 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1749 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1750 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1751 }

1752 for _, e := range l {

1753 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1754 return err

1755 }

1756 }

1757 for _, name := range mailboxes.Regular {

1758 if err := add(name, SpecialUse{}); err != nil {

1759 return err

1760 }

1761 }

1762 }

1763

1764 uidvalidity++

1765 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1766 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1767 }

1768 return nil

1769 })

1770}

1771

1772// Remove schedules an account for removal. New opens will fail. When the last

1773// reference is closed, the account files are removed.

1774func (a *Account) Remove(ctx context.Context) error {

1775 openAccounts.Lock()

1776 defer openAccounts.Unlock()

1777

1778 if err := AuthDB.Insert(ctx, &AccountRemove{AccountName: a.Name}); err != nil {

1779 return fmt.Errorf("inserting account removal: %w", err)

1780 }

1781 a.removed = true

1782

1783 return nil

1784}

1785

1786// WaitClosed waits until the last reference to this account is gone and the

1787// account is closed. Used during tests, to ensure the consistency checks run after

1788// expunged messages have been erased.

1789func (a *Account) WaitClosed() {

1790 <-a.closed

1791}

1792

1793// Close reduces the reference count, and closes the database connection when

1794// it was the last user.

1795func (a *Account) Close() error {

1796 if CheckConsistencyOnClose {

1797 xerr := a.CheckConsistency()

1798 err := closeAccount(a)

1799 if xerr != nil {

1800 panic(xerr)

1801 }

1802 return err

1803 }

1804 return closeAccount(a)

1805}

1806

1807// SetSkipMessageModSeqZeroCheck skips consistency checks for Message.ModSeq and

1808// Message.CreateSeq being zero.

1809func (a *Account) SetSkipMessageModSeqZeroCheck(skip bool) {

1810 a.Lock()

1811 defer a.Unlock()

1812 a.skipMessageZeroSeqCheck = true

1813}

1814

1815// CheckConsistency checks the consistency of the database and returns a non-nil

1816// error for these cases:

1817//

1818// - Missing or unexpected on-disk message files.

1819// - Mismatch between message size and length of MsgPrefix and on-disk file.

1820// - Incorrect mailbox counts.

1821// - Incorrect total message size.

1822// - Message with UID >= mailbox uid next.

1823// - Mailbox uidvalidity >= account uid validity.

1824// - Mailbox ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq, and Modseq >= highest message ModSeq.

1825// - Mailbox must have a live parent ID if they are live themselves, live names must be unique.

1826// - Message ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1827// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1828// - Annotations must have ModSeq > 0, CreateSeq > 0, ModSeq >= CreateSeq and live keys must be unique per mailbox.

1829// - Recalculate junk filter (words and counts) and check they are the same.

1830func (a *Account) CheckConsistency() error {

1831 a.Lock()

1832 defer a.Unlock()

1833

1834 var uidErrors []string // With a limit, could be many.

1835 var modseqErrors []string // With limit.

1836 var fileErrors []string // With limit.

1837 var threadidErrors []string // With limit.

1838 var threadParentErrors []string // With limit.

1839 var threadAncestorErrors []string // With limit.

1840 var errmsgs []string

1841

1842 ctx := context.Background()

1843 log := mlog.New("store", nil)

1844

1845 err := a.DB.Read(ctx, func(tx *bstore.Tx) error {

1846 nuv := NextUIDValidity{ID: 1}

1847 err := tx.Get(&nuv)

1848 if err != nil {

1849 return fmt.Errorf("fetching next uid validity: %v", err)

1850 }

1851

1852 mailboxes := map[int64]Mailbox{} // Also expunged mailboxes.

1853 mailboxNames := map[string]Mailbox{} // Only live names.

1854 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1855 mailboxes[mb.ID] = mb

1856 if !mb.Expunged {

1857 if xmb, ok := mailboxNames[mb.Name]; ok {

1858 errmsg := fmt.Sprintf("mailbox %q exists as id %d and id %d", mb.Name, mb.ID, xmb.ID)

1859 errmsgs = append(errmsgs, errmsg)

1860 }

1861 mailboxNames[mb.Name] = mb

1862 }

1863

1864 if mb.UIDValidity >= nuv.Next {

1865 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1866 errmsgs = append(errmsgs, errmsg)

1867 }

1868

1869 if mb.ModSeq == 0 || mb.CreateSeq == 0 || mb.CreateSeq > mb.ModSeq {

1870 errmsg := fmt.Sprintf("mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", mb.Name, mb.ID, mb.ModSeq, mb.CreateSeq)

1871 errmsgs = append(errmsgs, errmsg)

1872 return nil

1873 }

1874 m, err := bstore.QueryTx[Message](tx).FilterNonzero(Message{MailboxID: mb.ID}).SortDesc("ModSeq").Limit(1).Get()

1875 if err == bstore.ErrAbsent {

1876 return nil

1877 } else if err != nil {

1878 return fmt.Errorf("get message with highest modseq for mailbox: %v", err)

1879 } else if mb.ModSeq < m.ModSeq {

1880 errmsg := fmt.Sprintf("mailbox %q (id %d) has modseq %d < highest message modseq is %d", mb.Name, mb.ID, mb.ModSeq, m.ModSeq)

1881 errmsgs = append(errmsgs, errmsg)

1882 }

1883 return nil

1884 })

1885 if err != nil {

1886 return fmt.Errorf("checking mailboxes: %v", err)

1887 }

1888

1889 // Check ParentID and name of parent.

1890 for _, mb := range mailboxNames {

1891 if mox.ParentMailboxName(mb.Name) == "" {

1892 if mb.ParentID == 0 {

1893 continue

1894 }

1895 errmsg := fmt.Sprintf("mailbox %q (id %d) is a root mailbox but has parentid %d", mb.Name, mb.ID, mb.ParentID)

1896 errmsgs = append(errmsgs, errmsg)

1897 } else if mb.ParentID == 0 {

1898 errmsg := fmt.Sprintf("mailbox %q (id %d) is not a root mailbox but has a zero parentid", mb.Name, mb.ID)

1899 errmsgs = append(errmsgs, errmsg)

1900 } else if mox.ParentMailboxName(mb.Name) != mailboxes[mb.ParentID].Name {

1901 errmsg := fmt.Sprintf("mailbox %q (id %d) has parent mailbox id %d with name %q, but parent name should be %q", mb.Name, mb.ID, mb.ParentID, mailboxes[mb.ParentID].Name, mox.ParentMailboxName(mb.Name))

1902 errmsgs = append(errmsgs, errmsg)

1903 }

1904 }

1905

1906 type annotation struct {

1907 mailboxID int64 // Can be 0.

1908 key string

1909 }

1910 annotations := map[annotation]struct{}{}

1911 err = bstore.QueryTx[Annotation](tx).ForEach(func(a Annotation) error {

1912 if !a.Expunged {

1913 k := annotation{a.MailboxID, a.Key}

1914 if _, ok := annotations[k]; ok {

1915 errmsg := fmt.Sprintf("duplicate live annotation key %q for mailbox id %d", a.Key, a.MailboxID)

1916 errmsgs = append(errmsgs, errmsg)

1917 }

1918 annotations[k] = struct{}{}

1919 }

1920 if a.ModSeq == 0 || a.CreateSeq == 0 || a.CreateSeq > a.ModSeq {

1921 errmsg := fmt.Sprintf("annotation %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and modseq >= createseq", a.ID, mailboxes[a.MailboxID].Name, a.MailboxID, a.ModSeq, a.CreateSeq)

1922 errmsgs = append(errmsgs, errmsg)

1923 } else if a.MailboxID > 0 && mailboxes[a.MailboxID].ModSeq < a.ModSeq {

1924 errmsg := fmt.Sprintf("annotation %d in mailbox %q (id %d) has invalid modseq %d > mailbox modseq %d", a.ID, mailboxes[a.MailboxID].Name, a.MailboxID, a.ModSeq, mailboxes[a.MailboxID].ModSeq)

1925 errmsgs = append(errmsgs, errmsg)

1926 }

1927 return nil

1928 })

1929 if err != nil {

1930 return fmt.Errorf("checking mailbox annotations: %v", err)

1931 }

1932

1933 // All message id's from database. For checking for unexpected files afterwards.

1934 messageIDs := map[int64]struct{}{}

1935 eraseMessageIDs := map[int64]bool{} // Value indicates whether to skip updating disk usage.

1936

1937 // If configured, we'll be building up the junk filter for the messages, to compare

1938 // against the on-disk junk filter.

1939 var jf *junk.Filter

1940 conf, _ := a.Conf()

1941 if conf.JunkFilter != nil {

1942 random := make([]byte, 16)

1943 cryptorand.Read(random)

1944 dbpath := filepath.Join(mox.DataDirPath("tmp"), fmt.Sprintf("junkfilter-check-%x.db", random))

1945 bloompath := filepath.Join(mox.DataDirPath("tmp"), fmt.Sprintf("junkfilter-check-%x.bloom", random))

1946 os.MkdirAll(filepath.Dir(dbpath), 0700)

1947 defer func() {

1948 err := os.Remove(bloompath)

1949 log.Check(err, "removing temp bloom file")

1950 err = os.Remove(dbpath)

1951 log.Check(err, "removing temp junk filter database file")

1952 }()

1953 jf, err = junk.NewFilter(ctx, log, conf.JunkFilter.Params, dbpath, bloompath)

1954 if err != nil {

1955 return fmt.Errorf("new junk filter: %v", err)

1956 }

1957 defer func() {

1958 err := jf.Close()

1959 log.Check(err, "closing junk filter")

1960 }()

1961 }

1962 var ntrained int

1963

1964 // Get IDs of erase messages not yet removed, they'll have a message file.

1965 err = bstore.QueryTx[MessageErase](tx).ForEach(func(me MessageErase) error {

1966 eraseMessageIDs[me.ID] = me.SkipUpdateDiskUsage

1967 return nil

1968 })

1969 if err != nil {

1970 return fmt.Errorf("listing message erase records")

1971 }

1972

1973 counts := map[int64]MailboxCounts{}

1974 var totalExpungedSize int64

1975 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1976 mc := counts[m.MailboxID]

1977 mc.Add(m.MailboxCounts())

1978 counts[m.MailboxID] = mc

1979

1980 mb := mailboxes[m.MailboxID]

1981

1982 if (!a.skipMessageZeroSeqCheck && (m.ModSeq == 0 || m.CreateSeq == 0) || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1983 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1984 modseqErrors = append(modseqErrors, modseqerr)

1985 }

1986 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1987 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1988 uidErrors = append(uidErrors, uiderr)

1989 }

1990 if m.Expunged {

1991 if skip := eraseMessageIDs[m.ID]; !skip {

1992 totalExpungedSize += m.Size

1993 }

1994 return nil

1995 }

1996

1997 messageIDs[m.ID] = struct{}{}

1998 p := a.MessagePath(m.ID)

1999 st, err := os.Stat(p)

2000 if err != nil {

2001 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

2002 fileErrors = append(fileErrors, existserr)

2003 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

2004 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

2005 fileErrors = append(fileErrors, sizeerr)

2006 }

2007

2008 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

2009 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

2010 threadidErrors = append(threadidErrors, err)

2011 }

2012 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

2013 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

2014 threadParentErrors = append(threadParentErrors, err)

2015 }

2016 for i, pid := range m.ThreadParentIDs {

2017 am := Message{ID: pid}

2018 if err := tx.Get(&am); err == bstore.ErrAbsent || err == nil && am.Expunged {

2019 continue

2020 } else if err != nil {

2021 return fmt.Errorf("get ancestor message: %v", err)

2022 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

2023 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

2024 threadAncestorErrors = append(threadAncestorErrors, err)

2025 } else {

2026 break

2027 }

2028 }

2029

2030 if jf != nil {

2031 if m.Junk != m.Notjunk {

2032 ntrained++

2033 if _, err := a.TrainMessage(ctx, log, jf, m.Notjunk, m); err != nil {

2034 return fmt.Errorf("train message: %v", err)

2035 }

2036 // We are not setting m.TrainedJunk, we were only recalculating the words.

2037 }

2038 }

2039

2040 return nil

2041 })

2042 if err != nil {

2043 return fmt.Errorf("reading messages: %v", err)

2044 }

2045

2046 msgdir := filepath.Join(a.Dir, "msg")

2047 err = filepath.WalkDir(msgdir, func(path string, entry fs.DirEntry, err error) error {

2048 if err != nil {

2049 if path == msgdir && errors.Is(err, fs.ErrNotExist) {

2050 return nil

2051 }

2052 return err

2053 }

2054 if entry.IsDir() {

2055 return nil

2056 }

2057 id, err := strconv.ParseInt(filepath.Base(path), 10, 64)

2058 if err != nil {

2059 return fmt.Errorf("parsing message id from path %q: %v", path, err)

2060 }

2061 _, mok := messageIDs[id]

2062 _, meok := eraseMessageIDs[id]

2063 if !mok && !meok {

2064 return fmt.Errorf("unexpected message file %q", path)

2065 }

2066 return nil

2067 })

2068 if err != nil {

2069 return fmt.Errorf("walking message dir: %v", err)

2070 }

2071

2072 var totalMailboxSize int64

2073 for _, mb := range mailboxNames {

2074 totalMailboxSize += mb.Size

2075 if mb.MailboxCounts != counts[mb.ID] {

2076 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

2077 errmsgs = append(errmsgs, mbcounterr)

2078 }

2079 }

2080

2081 du := DiskUsage{ID: 1}

2082 if err := tx.Get(&du); err != nil {

2083 return fmt.Errorf("get diskusage")

2084 }

2085 if du.MessageSize != totalMailboxSize+totalExpungedSize {

2086 errmsg := fmt.Sprintf("total disk usage message size in database is %d != sum of mailbox message sizes %d + sum unerased expunged message sizes %d", du.MessageSize, totalMailboxSize, totalExpungedSize)

2087 errmsgs = append(errmsgs, errmsg)

2088 }

2089

2090 // Compare on-disk junk filter with our recalculated filter.

2091 if jf != nil {

2092 load := func(f *junk.Filter) (map[junk.Wordscore]struct{}, error) {

2093 words := map[junk.Wordscore]struct{}{}

2094 err := bstore.QueryDB[junk.Wordscore](ctx, f.DB()).ForEach(func(w junk.Wordscore) error {

2095 if w.Ham != 0 || w.Spam != 0 {

2096 words[w] = struct{}{}

2097 }

2098 return nil

2099 })

2100 if err != nil {

2101 return nil, fmt.Errorf("read junk filter wordscores: %v", err)

2102 }

2103 return words, nil

2104 }

2105 if err := jf.Save(); err != nil {

2106 return fmt.Errorf("save recalculated junk filter: %v", err)

2107 }

2108 wordsExp, err := load(jf)

2109 if err != nil {

2110 return fmt.Errorf("read recalculated junk filter: %v", err)

2111 }

2112

2113 ajf, _, err := a.OpenJunkFilter(ctx, log)

2114 if err != nil {

2115 return fmt.Errorf("open account junk filter: %v", err)

2116 }

2117 defer func() {

2118 err := ajf.Close()

2119 log.Check(err, "closing junk filter")

2120 }()

2121 wordsGot, err := load(ajf)

2122 if err != nil {

2123 return fmt.Errorf("read account junk filter: %v", err)

2124 }

2125

2126 if !reflect.DeepEqual(wordsGot, wordsExp) {

2127 errmsg := fmt.Sprintf("unexpected values in junk filter, trained %d of %d\ngot:\n%v\nexpected:\n%v", ntrained, len(messageIDs), wordsGot, wordsExp)

2128 errmsgs = append(errmsgs, errmsg)

2129 }

2130 }

2131

2132 return nil

2133 })

2134 if err != nil {

2135 return err

2136 }

2137 errmsgs = append(errmsgs, uidErrors...)

2138 errmsgs = append(errmsgs, modseqErrors...)

2139 errmsgs = append(errmsgs, fileErrors...)

2140 errmsgs = append(errmsgs, threadidErrors...)

2141 errmsgs = append(errmsgs, threadParentErrors...)

2142 errmsgs = append(errmsgs, threadAncestorErrors...)

2143 if len(errmsgs) > 0 {

2144 return fmt.Errorf("%s", strings.Join(errmsgs, "; "))

2145 }

2146 return nil

2147}

2148

2149// Conf returns the configuration for this account if it still exists. During

2150// an SMTP session, a configuration update may drop an account.

2151func (a *Account) Conf() (config.Account, bool) {

2152 return mox.Conf.Account(a.Name)

2153}

2154

2155// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

2156func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

2157 nuv := NextUIDValidity{ID: 1}

2158 if err := tx.Get(&nuv); err != nil {

2159 return 0, err

2160 }

2161 v := nuv.Next

2162 nuv.Next++

2163 if err := tx.Update(&nuv); err != nil {

2164 return 0, err

2165 }

2166 return v, nil

2167}

2168

2169// NextModSeq returns the next modification sequence, which is global per account,

2170// over all types.

2171func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

2172 return nextModSeq(tx)

2173}

2174

2175func nextModSeq(tx *bstore.Tx) (ModSeq, error) {

2176 v := SyncState{ID: 1}

2177 if err := tx.Get(&v); err == bstore.ErrAbsent {

2178 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

2179 // already used.

2180 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

2181 // makes sense.

2182 v = SyncState{1, 2, -1}

2183 return v.LastModSeq, tx.Insert(&v)

2184 } else if err != nil {

2185 return 0, err

2186 }

2187 v.LastModSeq++

2188 return v.LastModSeq, tx.Update(&v)

2189}

2190

2191func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

2192 v := SyncState{ID: 1}

2193 err := tx.Get(&v)

2194 if err == bstore.ErrAbsent {

2195 return 0, nil

2196 }

2197 return v.HighestDeletedModSeq, err

2198}

2199

2200// WithWLock runs fn with account writelock held. Necessary for account/mailbox

2201// modification. For message delivery, a read lock is required.

2202func (a *Account) WithWLock(fn func()) {

2203 a.Lock()

2204 defer a.Unlock()

2205 fn()

2206}

2207

2208// WithRLock runs fn with account read lock held. Needed for message delivery.

2209func (a *Account) WithRLock(fn func()) {

2210 a.RLock()

2211 defer a.RUnlock()

2212 fn()

2213}

2214

2215// AddOpts influence which work MessageAdd does. Some callers can batch

2216// checks/operations efficiently. For convenience and safety, a zero AddOpts does

2217// all the checks and work.

2218type AddOpts struct {

2219 SkipCheckQuota bool

2220

2221 // If set, the message size is not added to the disk usage. Caller must do that,

2222 // e.g. for many messages at once. If used together with SkipCheckQuota, the

2223 // DiskUsage is not read for database when adding a message.

2224 SkipUpdateDiskUsage bool

2225

2226 // Do not fsync the delivered message file. Useful when copying message files from

2227 // another mailbox. The hardlink created during delivery only needs a directory

2228 // fsync.

2229 SkipSourceFileSync bool

2230

2231 // The directory in which the message file is delivered, typically with a hard

2232 // link, is not fsynced. Useful when delivering many files. A single or few

2233 // directory fsyncs are more efficient.

2234 SkipDirSync bool

2235

2236 // Do not assign thread information to a message. Useful when importing many

2237 // messages and assigning threads efficiently after importing messages.

2238 SkipThreads bool

2239

2240 // If JunkFilter is set, it is used for training. If not set, and the filter must

2241 // be trained for a message, the junk filter is opened, modified and saved to disk.

2242 JunkFilter *junk.Filter

2243

2244 SkipTraining bool

2245

2246 // If true, a preview will be generated if the Message doesn't already have one.

2247 SkipPreview bool

2248}

2249

2250// todo optimization: when moving files, we open the original, call MessageAdd() which hardlinks it and close the file gain. when passing the filename, we could just use os.Link, saves 2 syscalls.

2251

2252// MessageAdd delivers a mail message to the account.

2253//

2254// The file is hardlinked or copied, the caller must clean up the original file. If

2255// this call succeeds, but the database transaction with the change can't be

2256// committed, the caller must clean up the delivered message file identified by

2257// m.ID.

2258//

2259// If the message does not fit in the quota, an error with ErrOverQuota is returned

2260// and the mailbox and message are unchanged and the transaction can continue. For

2261// other errors, the caller must abort the transaction.

2262//

2263// The message, with msg.MsgPrefix and msgFile combined, must have a header

2264// section. The caller is responsible for adding a header separator to

2265// msg.MsgPrefix if missing from an incoming message.

2266//

2267// If UID is not set, it is assigned automatically.

2268//

2269// If the message ModSeq is zero, it is assigned automatically. If the message

2270// CreateSeq is zero, it is set to ModSeq. The mailbox ModSeq is set to the message

2271// ModSeq.

2272//

2273// If the message does not fit in the quota, an error with ErrOverQuota is returned

2274// and the mailbox and message are unchanged and the transaction can continue. For

2275// other errors, the caller must abort the transaction.

2276//

2277// If the destination mailbox has the Sent special-use flag, the message is parsed

2278// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

2279// reputation classification.

2280//

2281// Must be called with account write lock held.

2282//

2283// Caller must save the mailbox after MessageAdd returns, and broadcast changes for

2284// new the message, updated mailbox counts and possibly new mailbox keywords.

2285func (a *Account) MessageAdd(log mlog.Log, tx *bstore.Tx, mb *Mailbox, m *Message, msgFile *os.File, opts AddOpts) (rerr error) {

2286 if m.Expunged {

2287 return fmt.Errorf("cannot deliver expunged message")

2288 }

2289

2290 if !opts.SkipUpdateDiskUsage || !opts.SkipCheckQuota {

2291 du := DiskUsage{ID: 1}

2292 if err := tx.Get(&du); err != nil {

2293 return fmt.Errorf("get disk usage: %v", err)

2294 }

2295

2296 if !opts.SkipCheckQuota {

2297 maxSize := a.QuotaMessageSize()

2298 if maxSize > 0 && m.Size > maxSize-du.MessageSize {

2299 return fmt.Errorf("%w: max size %d bytes", ErrOverQuota, maxSize)

2300 }

2301 }

2302

2303 if !opts.SkipUpdateDiskUsage {

2304 du.MessageSize += m.Size

2305 if err := tx.Update(&du); err != nil {

2306 return fmt.Errorf("update disk usage: %v", err)

2307 }

2308 }

2309 }

2310

2311 m.MailboxID = mb.ID

2312 if m.MailboxOrigID == 0 {

2313 m.MailboxOrigID = mb.ID

2314 }

2315 if m.UID == 0 {

2316 m.UID = mb.UIDNext

2317 if err := mb.UIDNextAdd(1); err != nil {

2318 return fmt.Errorf("adding uid: %v", err)

2319 }

2320 }

2321 if m.ModSeq == 0 {

2322 modseq, err := a.NextModSeq(tx)

2323 if err != nil {

2324 return fmt.Errorf("assigning next modseq: %w", err)

2325 }

2326 m.ModSeq = modseq

2327 } else if m.ModSeq < mb.ModSeq {

2328 return fmt.Errorf("cannot deliver message with modseq %d < mailbox modseq %d", m.ModSeq, mb.ModSeq)

2329 }

2330 if m.CreateSeq == 0 {

2331 m.CreateSeq = m.ModSeq

2332 }

2333 mb.ModSeq = m.ModSeq

2334

2335 if m.SaveDate == nil {

2336 now := time.Now()

2337 m.SaveDate = &now

2338 }

2339 if m.Received.IsZero() {

2340 m.Received = time.Now()

2341 }

2342

2343 if len(m.Keywords) > 0 {

2344 mb.Keywords, _ = MergeKeywords(mb.Keywords, m.Keywords)

2345 }

2346

2347 conf, _ := a.Conf()

2348 m.JunkFlagsForMailbox(*mb, conf)

2349

2350 var part *message.Part

2351 if m.ParsedBuf == nil {

2352 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

2353 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

2354 if err != nil {

2355 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

2356 // We continue, p is still valid.

2357 }

2358 part = &p

2359 buf, err := json.Marshal(part)

2360 if err != nil {

2361 return fmt.Errorf("marshal parsed message: %w", err)

2362 }

2363 m.ParsedBuf = buf

2364 }

2365

2366 var partTried bool

2367 getPart := func() *message.Part {

2368 if part != nil {

2369 return part

2370 }

2371 if partTried {

2372 return nil

2373 }

2374 partTried = true

2375 var p message.Part

2376 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

2377 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

2378 } else {

2379 mr := FileMsgReader(m.MsgPrefix, msgFile)

2380 p.SetReaderAt(mr)

2381 part = &p

2382 }

2383 return part

2384 }

2385

2386 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

2387 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

2388 m.MailboxDestinedID = 0

2389 }

2390

2391 if m.MessageID == "" && m.SubjectBase == "" && getPart() != nil {

2392 m.PrepareThreading(log, part)

2393 }

2394

2395 if !opts.SkipPreview && m.Preview == nil {

2396 if p := getPart(); p != nil {

2397 s, err := p.Preview(log)

2398 if err != nil {

2399 return fmt.Errorf("generating preview: %v", err)

2400 }

2401 m.Preview = &s

2402 }

2403 }

2404

2405 // Assign to thread (if upgrade has completed).

2406 noThreadID := opts.SkipThreads

2407 if m.ThreadID == 0 && !opts.SkipThreads && getPart() != nil {

2408 select {

2409 case <-a.threadsCompleted:

2410 if a.threadsErr != nil {

2411 log.Info("not assigning threads for new delivery, upgrading to threads failed")

2412 noThreadID = true

2413 } else {

2414 if err := assignThread(log, tx, m, part); err != nil {

2415 return fmt.Errorf("assigning thread: %w", err)

2416 }

2417 }

2418 default:

2419 // note: since we have a write transaction to get here, we can't wait for the

2420 // thread upgrade to finish.

2421 // If we don't assign a threadid the upgrade process will do it.

2422 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

2423 noThreadID = true

2424 }

2425 }

2426

2427 if err := tx.Insert(m); err != nil {

2428 return fmt.Errorf("inserting message: %w", err)

2429 }

2430 if !noThreadID && m.ThreadID == 0 {

2431 m.ThreadID = m.ID

2432 if err := tx.Update(m); err != nil {

2433 return fmt.Errorf("updating message for its own thread id: %w", err)

2434 }

2435 }

2436

2437 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's if the mail client doesn't save a message that includes the bcc header in the sent mailbox.

2438 if mb.Sent && getPart() != nil && part.Envelope != nil {

2439 e := part.Envelope

2440 sent := e.Date

2441 if sent.IsZero() {

2442 sent = m.Received

2443 }

2444 if sent.IsZero() {

2445 sent = time.Now()

2446 }

2447 addrs := append(append(e.To, e.CC...), e.BCC...)

2448 for _, addr := range addrs {

2449 if addr.User == "" {

2450 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

2451 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

2452 continue

2453 }

2454 d, err := dns.ParseDomain(addr.Host)

2455 if err != nil {

2456 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

2457 continue

2458 }

2459 lp, err := smtp.ParseLocalpart(addr.User)

2460 if err != nil {

2461 log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))

2462 continue

2463 }

2464 mr := Recipient{

2465 MessageID: m.ID,

2466 Localpart: lp.String(),

2467 Domain: d.Name(),

2468 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

2469 Sent: sent,

2470 }

2471 if err := tx.Insert(&mr); err != nil {

2472 return fmt.Errorf("inserting sent message recipients: %w", err)

2473 }

2474 }

2475 }

2476

2477 msgPath := a.MessagePath(m.ID)

2478 msgDir := filepath.Dir(msgPath)

2479 if a.lastMsgDir != msgDir {

2480 os.MkdirAll(msgDir, 0770)

2481 if err := moxio.SyncDir(log, msgDir); err != nil {

2482 return fmt.Errorf("sync message dir: %w", err)

2483 }

2484 a.lastMsgDir = msgDir

2485 }

2486

2487 // Sync file data to disk.

2488 if !opts.SkipSourceFileSync {

2489 if err := msgFile.Sync(); err != nil {

2490 return fmt.Errorf("fsync message file: %w", err)

2491 }

2492 }

2493

2494 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

2495 return fmt.Errorf("linking/copying message to new file: %w", err)

2496 }

2497

2498 defer func() {

2499 if rerr != nil {

2500 err := os.Remove(msgPath)

2501 log.Check(err, "removing delivered message file", slog.String("path", msgPath))

2502 }

2503 }()

2504

2505 if !opts.SkipDirSync {

2506 if err := moxio.SyncDir(log, msgDir); err != nil {

2507 return fmt.Errorf("sync directory: %w", err)

2508 }

2509 }

2510

2511 if !opts.SkipTraining && m.NeedsTraining() && a.HasJunkFilter() {

2512 jf, opened, err := a.ensureJunkFilter(context.TODO(), log, opts.JunkFilter)

2513 if err != nil {

2514 return fmt.Errorf("open junk filter: %w", err)

2515 }

2516 defer func() {

2517 if jf != nil && opened {

2518 err := jf.CloseDiscard()

2519 log.Check(err, "closing junk filter without saving")

2520 }

2521 }()

2522

2523 // todo optimize: should let us do the tx.Update of m if needed. we should at least merge it with the common case of setting a thread id. and we should try to merge that with the insert by expliciting getting the next id from bstore.

2524

2525 if err := a.RetrainMessage(context.TODO(), log, tx, jf, m); err != nil {

2526 return fmt.Errorf("training junkfilter: %w", err)

2527 }

2528

2529 if opened {

2530 err := jf.Close()

2531 jf = nil

2532 if err != nil {

2533 return fmt.Errorf("close junk filter: %w", err)

2534 }

2535 }

2536 }

2537

2538 mb.MailboxCounts.Add(m.MailboxCounts())

2539

2540 return nil

2541}

2542

2543// SetPassword saves a new password for this account. This password is used for

2544// IMAP, SMTP (submission) sessions and the HTTP account web page.

2545//

2546// Callers are responsible for checking if the account has NoCustomPassword set.

2547func (a *Account) SetPassword(log mlog.Log, password string) error {

2548 password, err := precis.OpaqueString.String(password)

2549 if err != nil {

2550 return fmt.Errorf(`password not allowed by "precis"`)

2551 }

2552

2553 if len(password) < 8 {

2554 // We actually check for bytes...

2555 return fmt.Errorf("password must be at least 8 characters long")

2556 }

2557

2558 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

2559 if err != nil {

2560 return fmt.Errorf("generating password hash: %w", err)

2561 }

2562

2563 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2564 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

2565 return fmt.Errorf("deleting existing password: %v", err)

2566 }

2567 var pw Password

2568 pw.Hash = string(hash)

2569

2570 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

2571 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

2572 // first block is based on the key/password. We hash those first blocks now, and

2573 // store the hash state in the database. When we actually authenticate, we'll

2574 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

2575 // because it does not expose its internal state and isn't a BinaryMarshaler.

2576 // ../rfc/2104:121

2577 pw.CRAMMD5.Ipad = md5.New()

2578 pw.CRAMMD5.Opad = md5.New()

2579 key := []byte(password)

2580 if len(key) > 64 {

2581 t := md5.Sum(key)

2582 key = t[:]

2583 }

2584 ipad := make([]byte, md5.BlockSize)

2585 opad := make([]byte, md5.BlockSize)

2586 copy(ipad, key)

2587 copy(opad, key)

2588 for i := range ipad {

2589 ipad[i] ^= 0x36

2590 opad[i] ^= 0x5c

2591 }

2592 pw.CRAMMD5.Ipad.Write(ipad)

2593 pw.CRAMMD5.Opad.Write(opad)

2594

2595 pw.SCRAMSHA1.Salt = scram.MakeRandom()

2596 pw.SCRAMSHA1.Iterations = 2 * 4096

2597 pw.SCRAMSHA1.SaltedPassword, err = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

2598 if err != nil {

2599 return fmt.Errorf("scram sha1 salt password: %w", err)

2600 }

2601

2602 pw.SCRAMSHA256.Salt = scram.MakeRandom()

2603 pw.SCRAMSHA256.Iterations = 4096

2604 pw.SCRAMSHA256.SaltedPassword, err = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

2605 if err != nil {

2606 return fmt.Errorf("scram sha256 salt password: %w", err)

2607 }

2608

2609 if err := tx.Insert(&pw); err != nil {

2610 return fmt.Errorf("inserting new password: %v", err)

2611 }

2612

2613 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

2614 })

2615 if err == nil {

2616 log.Info("new password set for account", slog.String("account", a.Name))

2617 }

2618 return err

2619}

2620

2621// SessionsClear invalidates all (web) login sessions for the account.

2622func (a *Account) SessionsClear(ctx context.Context, log mlog.Log) error {

2623 return a.DB.Write(ctx, func(tx *bstore.Tx) error {

2624 return sessionRemoveAll(ctx, log, tx, a.Name)

2625 })

2626}

2627

2628// Subjectpass returns the signing key for use with subjectpass for the given

2629// email address with canonical localpart.

2630func (a *Account) Subjectpass(email string) (key string, err error) {

2631 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2632 v := Subjectpass{Email: email}

2633 err := tx.Get(&v)

2634 if err == nil {

2635 key = v.Key

2636 return nil

2637 }

2638 if !errors.Is(err, bstore.ErrAbsent) {

2639 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

2640 }

2641 key = ""

2642 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

2643 buf := make([]byte, 16)

2644 cryptorand.Read(buf)

2645 for _, b := range buf {

2646 key += string(chars[int(b)%len(chars)])

2647 }

2648 v.Key = key

2649 return tx.Insert(&v)

2650 })

2651}

2652

2653// Ensure mailbox is present in database, adding records for the mailbox and its

2654// parents if they aren't present.

2655//

2656// If subscribe is true, any mailboxes that were created will also be subscribed to.

2657//

2658// The leaf mailbox is created with special-use flags, taking the flags away from

2659// other mailboxes, and reflecting that in the returned changes.

2660//

2661// Modseq is used, and initialized if 0, for created mailboxes.

2662//

2663// Name must be in normalized form, see config.CheckMailboxName.

2664//

2665// Caller must hold account wlock.

2666// Caller must propagate changes if any.

2667func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, specialUse SpecialUse, modseq *ModSeq) (mb Mailbox, changes []Change, rerr error) {

2668 if norm.NFC.String(name) != name {

2669 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

2670 }

2671

2672 // Quick sanity check.

2673 if strings.EqualFold(name, "inbox") && name != "Inbox" {

2674 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

2675 }

2676

2677 // Get mailboxes with same name or prefix (parents).

2678 elems := strings.Split(name, "/")

2679 q := bstore.QueryTx[Mailbox](tx)

2680 q.FilterEqual("Expunged", false)

2681 q.FilterFn(func(xmb Mailbox) bool {

2682 t := strings.Split(xmb.Name, "/")

2683 return len(t) <= len(elems) && slices.Equal(t, elems[:len(t)])

2684 })

2685 l, err := q.List()

2686 if err != nil {

2687 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

2688 }

2689

2690 mailboxes := map[string]Mailbox{}

2691 for _, xmb := range l {

2692 mailboxes[xmb.Name] = xmb

2693 }

2694

2695 p := ""

2696 var exists bool

2697 var parentID int64

2698 for _, elem := range elems {

2699 if p != "" {

2700 p += "/"

2701 }

2702 p += elem

2703 mb, exists = mailboxes[p]

2704 if exists {

2705 parentID = mb.ID

2706 continue

2707 }

2708 uidval, err := a.NextUIDValidity(tx)

2709 if err != nil {

2710 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

2711 }

2712 if *modseq == 0 {

2713 *modseq, err = a.NextModSeq(tx)

2714 if err != nil {

2715 return Mailbox{}, nil, fmt.Errorf("next modseq: %v", err)

2716 }

2717 }

2718 mb = Mailbox{

2719 CreateSeq: *modseq,

2720 ModSeq: *modseq,

2721 ParentID: parentID,

2722 Name: p,

2723 UIDValidity: uidval,

2724 UIDNext: 1,

2725 HaveCounts: true,

2726 }

2727 err = tx.Insert(&mb)

2728 if err != nil {

2729 return Mailbox{}, nil, fmt.Errorf("creating new mailbox %q: %v", p, err)

2730 }

2731 parentID = mb.ID

2732

2733 var flags []string

2734 if subscribe {

2735 if tx.Get(&Subscription{p}) != nil {

2736 err := tx.Insert(&Subscription{p})

2737 if err != nil {

2738 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox %q: %v", p, err)

2739 }

2740 }

2741 flags = []string{`\Subscribed`}

2742 } else if err := tx.Get(&Subscription{p}); err == nil {

2743 flags = []string{`\Subscribed`}

2744 } else if err != bstore.ErrAbsent {

2745 return Mailbox{}, nil, fmt.Errorf("looking up subscription for %q: %v", p, err)

2746 }

2747

2748 changes = append(changes, ChangeAddMailbox{mb, flags})

2749 }

2750

2751 // Clear any special-use flags from existing mailboxes and assign them to this mailbox.

2752 var zeroSpecialUse SpecialUse

2753 if !exists && specialUse != zeroSpecialUse {

2754 var qerr error

2755 clearSpecialUse := func(b bool, fn func(*Mailbox) *bool) {

2756 if !b || qerr != nil {

2757 return

2758 }

2759 qs := bstore.QueryTx[Mailbox](tx)

2760 qs.FilterFn(func(xmb Mailbox) bool {

2761 return *fn(&xmb)

2762 })

2763 xmb, err := qs.Get()

2764 if err == bstore.ErrAbsent {

2765 return

2766 } else if err != nil {

2767 qerr = fmt.Errorf("looking up mailbox with special-use flag: %v", err)

2768 return

2769 }

2770 p := fn(&xmb)

2771 *p = false

2772 xmb.ModSeq = *modseq

2773 if err := tx.Update(&xmb); err != nil {

2774 qerr = fmt.Errorf("clearing special-use flag: %v", err)

2775 } else {

2776 changes = append(changes, xmb.ChangeSpecialUse())

2777 }

2778 }

2779 clearSpecialUse(specialUse.Archive, func(xmb *Mailbox) *bool { return &xmb.Archive })

2780 clearSpecialUse(specialUse.Draft, func(xmb *Mailbox) *bool { return &xmb.Draft })

2781 clearSpecialUse(specialUse.Junk, func(xmb *Mailbox) *bool { return &xmb.Junk })

2782 clearSpecialUse(specialUse.Sent, func(xmb *Mailbox) *bool { return &xmb.Sent })

2783 clearSpecialUse(specialUse.Trash, func(xmb *Mailbox) *bool { return &xmb.Trash })

2784 if qerr != nil {

2785 return Mailbox{}, nil, qerr

2786 }

2787

2788 mb.SpecialUse = specialUse

2789 mb.ModSeq = *modseq

2790 if err := tx.Update(&mb); err != nil {

2791 return Mailbox{}, nil, fmt.Errorf("setting special-use flag for new mailbox: %v", err)

2792 }

2793 changes = append(changes, mb.ChangeSpecialUse())

2794 }

2795 return mb, changes, nil

2796}

2797

2798// MailboxExists checks if mailbox exists.

2799// Caller must hold account rlock.

2800func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

2801 q := bstore.QueryTx[Mailbox](tx)

2802 q.FilterEqual("Expunged", false)

2803 q.FilterEqual("Name", name)

2804 return q.Exists()

2805}

2806

2807// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

2808func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

2809 q := bstore.QueryTx[Mailbox](tx)

2810 q.FilterEqual("Expunged", false)

2811 q.FilterEqual("Name", name)

2812 mb, err := q.Get()

2813 if err == bstore.ErrAbsent {

2814 return nil, nil

2815 }

2816 if err != nil {

2817 return nil, fmt.Errorf("looking up mailbox: %w", err)

2818 }

2819 return &mb, nil

2820}

2821

2822// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

2823// have to exist. Any parents are not automatically subscribed.

2824// Changes are returned and must be broadcasted by the caller.

2825func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

2826 if err := tx.Get(&Subscription{name}); err == nil {

2827 return nil, nil

2828 }

2829

2830 if err := tx.Insert(&Subscription{name}); err != nil {

2831 return nil, fmt.Errorf("inserting subscription: %w", err)

2832 }

2833

2834 q := bstore.QueryTx[Mailbox](tx)

2835 q.FilterEqual("Expunged", false)

2836 q.FilterEqual("Name", name)

2837 _, err := q.Get()

2838 if err == nil {

2839 return []Change{ChangeAddSubscription{name, nil}}, nil

2840 } else if err != bstore.ErrAbsent {

2841 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

2842 }

2843 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

2844}

2845

2846// MessageRuleset returns the first ruleset (if any) that matches the message

2847// represented by msgPrefix and msgFile, with smtp and validation fields from m.

2848func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

2849 if len(dest.Rulesets) == 0 {

2850 return nil

2851 }

2852

2853 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

2854 p, err := message.Parse(log.Logger, false, mr)

2855 if err != nil {

2856 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

2857 // note: part is still set.

2858 }

2859 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

2860 header, err := p.Header()

2861 if err != nil {

2862 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

2863 // todo: reject message?

2864 return nil

2865 }

2866

2867ruleset:

2868 for _, rs := range dest.Rulesets {

2869 if rs.SMTPMailFromRegexpCompiled != nil {

2870 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

2871 continue ruleset

2872 }

2873 }

2874 if rs.MsgFromRegexpCompiled != nil {

2875 if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {

2876 continue ruleset

2877 }

2878 }

2879

2880 if !rs.VerifiedDNSDomain.IsZero() {

2881 d := rs.VerifiedDNSDomain.Name()

2882 suffix := "." + d

2883 matchDomain := func(s string) bool {

2884 return s == d || strings.HasSuffix(s, suffix)

2885 }

2886 var ok bool

2887 if m.EHLOValidated && matchDomain(m.EHLODomain) {

2888 ok = true

2889 }

2890 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

2891 ok = true

2892 }

2893 if slices.ContainsFunc(m.DKIMDomains, matchDomain) {

2894 ok = true

2895 }

2896 if !ok {

2897 continue ruleset

2898 }

2899 }

2900

2901 header:

2902 for _, t := range rs.HeadersRegexpCompiled {

2903 for k, vl := range header {

2904 k = strings.ToLower(k)

2905 if !t[0].MatchString(k) {

2906 continue

2907 }

2908 for _, v := range vl {

2909 v = strings.ToLower(strings.TrimSpace(v))

2910 if t[1].MatchString(v) {

2911 continue header

2912 }

2913 }

2914 }

2915 continue ruleset

2916 }

2917 return &rs

2918 }

2919 return nil

2920}

2921

2922// MessagePath returns the file system path of a message.

2923func (a *Account) MessagePath(messageID int64) string {

2924 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

2925}

2926

2927// MessageReader opens a message for reading, transparently combining the

2928// message prefix with the original incoming message.

2929func (a *Account) MessageReader(m Message) *MsgReader {

2930 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

2931}

2932

2933// DeliverDestination delivers an email to dest, based on the configured rulesets.

2934//

2935// Returns ErrOverQuota when account would be over quota after adding message.

2936//

2937// Caller must hold account wlock (mailbox may be created).

2938// Message delivery, possible mailbox creation, and updated mailbox counts are

2939// broadcasted.

2940func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

2941 var mailbox string

2942 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

2943 if rs != nil {

2944 mailbox = rs.Mailbox

2945 } else if dest.Mailbox == "" {

2946 mailbox = "Inbox"

2947 } else {

2948 mailbox = dest.Mailbox

2949 }

2950 return a.DeliverMailbox(log, mailbox, "", m, msgFile)

2951}

2952

2953// DeliverMailbox delivers an email to mailbox while recording mailboxDestined as

2954// the mailbox to which it would normally have been delivered (if it weren't for

2955// introbox and/or rejects). If mailboxDestined is empty, or equal to mailbox, the

2956// effect is a regular delivery to mailbox. Both mailboxes are created if

2957// necessary.

2958//

2959// Returns ErrOverQuota when account would be over quota after adding message.

2960//

2961// Caller must hold account wlock (mailbox may be created).

2962//

2963// Message delivery, possible mailbox creation, and updated mailbox counts are

2964// broadcasted.

2965func (a *Account) DeliverMailbox(log mlog.Log, mailbox, mailboxDestined string, m *Message, msgFile *os.File) (rerr error) {

2966 var changes []Change

2967

2968 var commit bool

2969 defer func() {

2970 if !commit && m.ID != 0 {

2971 p := a.MessagePath(m.ID)

2972 err := os.Remove(p)

2973 log.Check(err, "remove delivered message file", slog.String("path", p))

2974 m.ID = 0

2975 }

2976 }()

2977

2978 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2979 if mailboxDestined != "" && mailboxDestined != mailbox {

2980 mbDestined, chl, err := a.MailboxEnsure(tx, mailboxDestined, true, SpecialUse{}, &m.ModSeq)

2981 if err != nil {

2982 return fmt.Errorf("ensuring intended mailbox: %w", err)

2983 }

2984 m.MailboxDestinedID = mbDestined.ID

2985 changes = append(changes, chl...)

2986 }

2987

2988 mb, chl, err := a.MailboxEnsure(tx, mailbox, true, SpecialUse{}, &m.ModSeq)

2989 if err != nil {

2990 return fmt.Errorf("ensuring mailbox: %w", err)

2991 }

2992 if m.CreateSeq == 0 {

2993 m.CreateSeq = m.ModSeq

2994 }

2995

2996 nmbkeywords := len(mb.Keywords)

2997

2998 if err := a.MessageAdd(log, tx, &mb, m, msgFile, AddOpts{}); err != nil {

2999 return err

3000 }

3001

3002 if err := tx.Update(&mb); err != nil {

3003 return fmt.Errorf("updating mailbox for delivery: %w", err)

3004 }

3005

3006 changes = append(changes, chl...)

3007 changes = append(changes, m.ChangeAddUID(mb), mb.ChangeCounts())

3008 if nmbkeywords != len(mb.Keywords) {

3009 changes = append(changes, mb.ChangeKeywords())

3010 }

3011 return nil

3012 })

3013 if err != nil {

3014 return err

3015 }

3016 commit = true

3017 BroadcastChanges(a, changes)

3018 return nil

3019}

3020

3021type RemoveOpts struct {

3022 JunkFilter *junk.Filter // If set, this filter is used for training, instead of opening and saving the junk filter.

3023}

3024

3025// MessageRemove markes messages as expunged, updates mailbox counts for the

3026// messages, sets a new modseq on the messages and mailbox, untrains the junk

3027// filter and queues the messages for erasing when the last reference has gone.

3028//

3029// Caller must save the modified mailbox to the database.

3030//

3031// The disk usage is not immediately updated. That will happen when the message

3032// is actually removed from disk.

3033//

3034// The junk filter is untrained for the messages if it was trained.

3035// Useful as optimization when messages are moved and the junk/nonjunk flags do not

3036// change (which can happen due to automatic junk/nonjunk flags for mailboxes).

3037//

3038// An empty list of messages results in an error.

3039//

3040// Caller must broadcast changes.

3041//

3042// Must be called with wlock held.

3043func (a *Account) MessageRemove(log mlog.Log, tx *bstore.Tx, modseq ModSeq, mb *Mailbox, opts RemoveOpts, l ...Message) (chremuids ChangeRemoveUIDs, chmbc ChangeMailboxCounts, rerr error) {

3044 if len(l) == 0 {

3045 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("must expunge at least one message")

3046 }

3047

3048 mb.ModSeq = modseq

3049

3050 // Remove any message recipients.

3051 anyIDs := make([]any, len(l))

3052 for i, m := range l {

3053 anyIDs[i] = m.ID

3054 }

3055 qmr := bstore.QueryTx[Recipient](tx)

3056 qmr.FilterEqual("MessageID", anyIDs...)

3057 if _, err := qmr.Delete(); err != nil {

3058 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("deleting message recipients for messages: %w", err)

3059 }

3060

3061 // Loaded lazily.

3062 jf := opts.JunkFilter

3063

3064 // Mark messages expunged.

3065 ids := make([]int64, 0, len(l))

3066 uids := make([]UID, 0, len(l))

3067 for _, m := range l {

3068 ids = append(ids, m.ID)

3069 uids = append(uids, m.UID)

3070

3071 if m.Expunged {

3072 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("message %d is already expunged", m.ID)

3073 }

3074

3075 mb.Sub(m.MailboxCounts())

3076

3077 m.ModSeq = modseq

3078 m.Expunged = true

3079 m.Junk = false

3080 m.Notjunk = false

3081

3082 if err := tx.Update(&m); err != nil {

3083 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("marking message %d expunged: %v", m.ID, err)

3084 }

3085

3086 // Ensure message gets erased in future.

3087 if err := tx.Insert(&MessageErase{m.ID, false}); err != nil {

3088 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("inserting message erase %d : %v", m.ID, err)

3089 }

3090

3091 if m.TrainedJunk == nil || !a.HasJunkFilter() {

3092 continue

3093 }

3094 // Untrain, as needed by updated flags Junk/Notjunk to false.

3095 if jf == nil {

3096 var err error

3097 jf, _, err = a.OpenJunkFilter(context.TODO(), log)

3098 if err != nil {

3099 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("open junk filter: %v", err)

3100 }

3101 defer func() {

3102 err := jf.Close()

3103 if rerr == nil {

3104 rerr = err

3105 } else {

3106 log.Check(err, "closing junk filter")

3107 }

3108 }()

3109 }

3110 if err := a.RetrainMessage(context.TODO(), log, tx, jf, &m); err != nil {

3111 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("retraining expunged messages: %w", err)

3112 }

3113 }

3114

3115 return ChangeRemoveUIDs{mb.ID, uids, modseq, ids, mb.UIDNext, mb.MessageCountIMAP(), uint32(mb.MailboxCounts.Unseen)}, mb.ChangeCounts(), nil

3116}

3117

3118// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

3119//

3120// The changed mailbox is saved to the database.

3121//

3122// Caller most hold account wlock.

3123// Caller must broadcast changes.

3124func (a *Account) TidyRejectsMailbox(log mlog.Log, tx *bstore.Tx, mbRej *Mailbox) (changes []Change, hasSpace bool, rerr error) {

3125 // Gather old messages to expunge.

3126 old := time.Now().Add(-14 * 24 * time.Hour)

3127 qdel := bstore.QueryTx[Message](tx)

3128 qdel.FilterNonzero(Message{MailboxID: mbRej.ID})

3129 qdel.FilterEqual("Expunged", false)

3130 qdel.FilterLess("Received", old)

3131 qdel.SortAsc("UID")

3132 expunge, err := qdel.List()

3133 if err != nil {

3134 return nil, false, fmt.Errorf("listing old messages: %w", err)

3135 }

3136

3137 if len(expunge) > 0 {

3138 modseq, err := a.NextModSeq(tx)

3139 if err != nil {

3140 return nil, false, fmt.Errorf("next mod seq: %v", err)

3141 }

3142

3143 chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mbRej, RemoveOpts{}, expunge...)

3144 if err != nil {

3145 return nil, false, fmt.Errorf("removing messages: %w", err)

3146 }

3147 if err := tx.Update(mbRej); err != nil {

3148 return nil, false, fmt.Errorf("updating mailbox: %v", err)

3149 }

3150 changes = append(changes, chremuids, chmbcounts)

3151 }

3152

3153 // We allow up to n messages.

3154 qcount := bstore.QueryTx[Message](tx)

3155 qcount.FilterNonzero(Message{MailboxID: mbRej.ID})

3156 qcount.FilterEqual("Expunged", false)

3157 qcount.Limit(1000)

3158 n, err := qcount.Count()

3159 if err != nil {

3160 return nil, false, fmt.Errorf("counting rejects: %w", err)

3161 }

3162 hasSpace = n < 1000

3163

3164 return changes, hasSpace, nil

3165}

3166

3167// RejectsRemove removes a message from the rejects mailbox if present.

3168//

3169// Caller most hold account wlock.

3170// Changes are broadcasted.

3171func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

3172 var changes []Change

3173

3174 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

3175 mb, err := a.MailboxFind(tx, rejectsMailbox)

3176 if err != nil {

3177 return fmt.Errorf("finding mailbox: %w", err)

3178 }

3179 if mb == nil {

3180 return nil

3181 }

3182

3183 q := bstore.QueryTx[Message](tx)

3184 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

3185 q.FilterEqual("Expunged", false)

3186 expunge, err := q.List()

3187 if err != nil {

3188 return fmt.Errorf("listing messages to remove: %w", err)

3189 }

3190

3191 if len(expunge) == 0 {

3192 return nil

3193 }

3194

3195 modseq, err := a.NextModSeq(tx)

3196 if err != nil {

3197 return fmt.Errorf("get next mod seq: %v", err)

3198 }

3199

3200 chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mb, RemoveOpts{}, expunge...)

3201 if err != nil {

3202 return fmt.Errorf("removing messages: %w", err)

3203 }

3204 changes = append(changes, chremuids, chmbcounts)

3205

3206 if err := tx.Update(mb); err != nil {

3207 return fmt.Errorf("saving mailbox: %w", err)

3208 }

3209

3210 return nil

3211 })

3212 if err != nil {

3213 return err

3214 }

3215

3216 BroadcastChanges(a, changes)

3217

3218 return nil

3219}

3220

3221// AddMessageSize adjusts the DiskUsage.MessageSize by size.

3222func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

3223 du := DiskUsage{ID: 1}

3224 if err := tx.Get(&du); err != nil {

3225 return fmt.Errorf("get diskusage: %v", err)

3226 }

3227 du.MessageSize += size

3228 if du.MessageSize < 0 {

3229 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

3230 }

3231 if err := tx.Update(&du); err != nil {

3232 return fmt.Errorf("update total message size: %v", err)

3233 }

3234 return nil

3235}

3236

3237// QuotaMessageSize returns the effective maximum total message size for an

3238// account. Returns 0 if there is no maximum.

3239func (a *Account) QuotaMessageSize() int64 {

3240 conf, _ := a.Conf()

3241 size := conf.QuotaMessageSize

3242 if size == 0 {

3243 size = mox.Conf.Static.QuotaMessageSize

3244 }

3245 if size < 0 {

3246 size = 0

3247 }

3248 return size

3249}

3250

3251// CanAddMessageSize checks if a message of size bytes can be added, depending on

3252// total message size and configured quota for account.

3253func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

3254 maxSize = a.QuotaMessageSize()

3255 if maxSize <= 0 {

3256 return true, 0, nil

3257 }

3258

3259 du := DiskUsage{ID: 1}

3260 if err := tx.Get(&du); err != nil {

3261 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

3262 }

3263 return du.MessageSize+size <= maxSize, maxSize, nil

3264}

3265

3266// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

3267var authCache = struct {

3268 sync.Mutex

3269 success map[authKey]string

3270}{

3271 success: map[authKey]string{},

3272}

3273

3274type authKey struct {

3275 email, hash string

3276}

3277

3278// StartAuthCache starts a goroutine that regularly clears the auth cache.

3279func StartAuthCache() {

3280 go manageAuthCache()

3281}

3282

3283func manageAuthCache() {

3284 for {

3285 authCache.Lock()

3286 authCache.success = map[authKey]string{}

3287 authCache.Unlock()

3288 time.Sleep(15 * time.Minute)

3289 }

3290}

3291

3292// OpenEmailAuth opens an account given an email address and password.

3293//

3294// The email address may contain a catchall separator.

3295// For invalid credentials, a nil account is returned, but accName may be

3296// non-empty.

3297func OpenEmailAuth(log mlog.Log, email string, password string, checkLoginDisabled bool) (racc *Account, raccName string, rerr error) {

3298 // We check for LoginDisabled after verifying the password. Otherwise users can get

3299 // messages about the account being disabled without knowing the password.

3300 acc, accName, _, err := OpenEmail(log, email, false)

3301 if err != nil {

3302 return nil, "", err

3303 }

3304

3305 defer func() {

3306 if rerr != nil {

3307 err := acc.Close()

3308 log.Check(err, "closing account after open auth failure")

3309 acc = nil

3310 }

3311 }()

3312

3313 password, err = precis.OpaqueString.String(password)

3314 if err != nil {

3315 return nil, "", ErrUnknownCredentials

3316 }

3317

3318 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

3319 if err != nil {

3320 if err == bstore.ErrAbsent {

3321 return nil, "", ErrUnknownCredentials

3322 }

3323 return nil, "", fmt.Errorf("looking up password: %v", err)

3324 }

3325 authCache.Lock()

3326 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

3327 authCache.Unlock()

3328 if !ok {

3329 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

3330 return nil, "", ErrUnknownCredentials

3331 }

3332 }

3333 if checkLoginDisabled {

3334 conf, aok := acc.Conf()

3335 if !aok {

3336 return nil, "", fmt.Errorf("cannot find config for account")

3337 } else if conf.LoginDisabled != "" {

3338 return nil, "", fmt.Errorf("%w: %s", ErrLoginDisabled, conf.LoginDisabled)

3339 }

3340 }

3341 authCache.Lock()

3342 authCache.success[authKey{email, pw.Hash}] = password

3343 authCache.Unlock()

3344 return acc, accName, nil

3345}

3346

3347// OpenEmail opens an account given an email address.

3348//

3349// The email address may contain a catchall separator.

3350//

3351// Returns account on success, may return non-empty account name even on error.

3352func OpenEmail(log mlog.Log, email string, checkLoginDisabled bool) (*Account, string, config.Destination, error) {

3353 addr, err := smtp.ParseAddress(email)

3354 if err != nil {

3355 return nil, "", config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

3356 }

3357 accountName, _, _, dest, err := mox.LookupAddress(addr.Localpart, addr.Domain, false, false, false)

3358 if err != nil && (errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

3359 return nil, accountName, config.Destination{}, ErrUnknownCredentials

3360 } else if err != nil {

3361 return nil, accountName, config.Destination{}, fmt.Errorf("looking up address: %v", err)

3362 }

3363 acc, err := OpenAccount(log, accountName, checkLoginDisabled)

3364 if err != nil {

3365 return nil, accountName, config.Destination{}, err

3366 }

3367 return acc, accountName, dest, nil

3368}

3369

3370// We store max 1<<shift files in each subdir of an account "msg" directory.

3371// Defaults to 1 for easy use in tests. Set to 13, for 8k message files, in main

3372// for normal operation.

3373var msgFilesPerDirShift = 1

3374var msgFilesPerDir int64 = 1 << msgFilesPerDirShift

3375

3376func MsgFilesPerDirShiftSet(shift int) {

3377 msgFilesPerDirShift = shift

3378 msgFilesPerDir = 1 << shift

3379}

3380

3381// 64 characters, must be power of 2 for MessagePath

3382const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

3383

3384// MessagePath returns the filename of the on-disk filename, relative to the

3385// containing directory such as <account>/msg or queue.

3386// Returns names like "AB/1".

3387func MessagePath(messageID int64) string {

3388 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

3389}

3390

3391// messagePathElems returns the elems, for a single join without intermediate

3392// string allocations.

3393func messagePathElems(messageID int64) []string {

3394 v := messageID >> msgFilesPerDirShift

3395 var dir strings.Builder

3396 for {

3397 dir.WriteString(string(msgDirChars[int(v)&(len(msgDirChars)-1)]))

3398 v >>= 6

3399 if v == 0 {

3400 break

3401 }

3402 }

3403 return []string{dir.String(), strconv.FormatInt(messageID, 10)}

3404}

3405

3406// Set returns a copy of f, with each flag that is true in mask set to the

3407// value from flags.

3408func (f Flags) Set(mask, flags Flags) Flags {

3409 set := func(d *bool, m, v bool) {

3410 if m {

3411 *d = v

3412 }

3413 }

3414 r := f

3415 set(&r.Seen, mask.Seen, flags.Seen)

3416 set(&r.Answered, mask.Answered, flags.Answered)

3417 set(&r.Flagged, mask.Flagged, flags.Flagged)

3418 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

3419 set(&r.Junk, mask.Junk, flags.Junk)

3420 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

3421 set(&r.Deleted, mask.Deleted, flags.Deleted)

3422 set(&r.Draft, mask.Draft, flags.Draft)

3423 set(&r.Phishing, mask.Phishing, flags.Phishing)

3424 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

3425 return r

3426}

3427

3428// Changed returns a mask of flags that have been between f and other.

3429func (f Flags) Changed(other Flags) (mask Flags) {

3430 mask.Seen = f.Seen != other.Seen

3431 mask.Answered = f.Answered != other.Answered

3432 mask.Flagged = f.Flagged != other.Flagged

3433 mask.Forwarded = f.Forwarded != other.Forwarded

3434 mask.Junk = f.Junk != other.Junk

3435 mask.Notjunk = f.Notjunk != other.Notjunk

3436 mask.Deleted = f.Deleted != other.Deleted

3437 mask.Draft = f.Draft != other.Draft

3438 mask.Phishing = f.Phishing != other.Phishing

3439 mask.MDNSent = f.MDNSent != other.MDNSent

3440 return

3441}

3442

3443// Strings returns the flags that are set in their string form.

3444func (f Flags) Strings() []string {

3445 fields := []struct {

3446 word string

3447 have bool

3448 }{

3449 {`$forwarded`, f.Forwarded},

3450 {`$junk`, f.Junk},

3451 {`$mdnsent`, f.MDNSent},

3452 {`$notjunk`, f.Notjunk},

3453 {`$phishing`, f.Phishing},

3454 {`\answered`, f.Answered},

3455 {`\deleted`, f.Deleted},

3456 {`\draft`, f.Draft},

3457 {`\flagged`, f.Flagged},

3458 {`\seen`, f.Seen},

3459 }

3460 var l []string

3461 for _, fh := range fields {

3462 if fh.have {

3463 l = append(l, fh.word)

3464 }

3465 }

3466 return l

3467}

3468

3469var systemWellKnownFlags = map[string]bool{

3470 `\answered`: true,

3471 `\flagged`: true,

3472 `\deleted`: true,

3473 `\seen`: true,

3474 `\draft`: true,

3475 `$junk`: true,

3476 `$notjunk`: true,

3477 `$forwarded`: true,

3478 `$phishing`: true,

3479 `$mdnsent`: true,

3480}

3481

3482// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

3483// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

3484func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

3485 fields := map[string]*bool{

3486 `\answered`: &flags.Answered,

3487 `\flagged`: &flags.Flagged,

3488 `\deleted`: &flags.Deleted,

3489 `\seen`: &flags.Seen,

3490 `\draft`: &flags.Draft,

3491 `$junk`: &flags.Junk,

3492 `$notjunk`: &flags.Notjunk,

3493 `$forwarded`: &flags.Forwarded,

3494 `$phishing`: &flags.Phishing,

3495 `$mdnsent`: &flags.MDNSent,

3496 }

3497 seen := map[string]bool{}

3498 for _, f := range l {

3499 f = strings.ToLower(f)

3500 if field, ok := fields[f]; ok {

3501 *field = true

3502 } else if seen[f] {

3503 if mox.Pedantic {

3504 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

3505 }

3506 } else {

3507 if err := CheckKeyword(f); err != nil {

3508 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

3509 }

3510 keywords = append(keywords, f)

3511 seen[f] = true

3512 }

3513 }

3514 sort.Strings(keywords)

3515 return flags, keywords, nil

3516}

3517

3518// RemoveKeywords removes keywords from l, returning whether any modifications were

3519// made, and a slice, a new slice in case of modifications. Keywords must have been

3520// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

3521// be used with valid keywords, not with system flags like \Seen.

3522func RemoveKeywords(l, remove []string) ([]string, bool) {

3523 var copied bool

3524 var changed bool

3525 for _, k := range remove {

3526 if i := slices.Index(l, k); i >= 0 {

3527 if !copied {

3528 l = slices.Clone(l)

3529 copied = true

3530 }

3531 copy(l[i:], l[i+1:])

3532 l = l[:len(l)-1]

3533 changed = true

3534 }

3535 }

3536 return l, changed

3537}

3538

3539// MergeKeywords adds keywords from add into l, returning whether it added any

3540// keyword, and the slice with keywords, a new slice if modifications were made.

3541// Keywords are only added if they aren't already present. Should only be used with

3542// keywords, not with system flags like \Seen.

3543func MergeKeywords(l, add []string) ([]string, bool) {

3544 var copied bool

3545 var changed bool

3546 for _, k := range add {

3547 if !slices.Contains(l, k) {

3548 if !copied {

3549 l = slices.Clone(l)

3550 copied = true

3551 }

3552 l = append(l, k)

3553 changed = true

3554 }

3555 }

3556 if changed {

3557 sort.Strings(l)

3558 }

3559 return l, changed

3560}

3561

3562// CheckKeyword returns an error if kw is not a valid keyword. Kw should

3563// already be in lower-case.

3564func CheckKeyword(kw string) error {

3565 if kw == "" {

3566 return fmt.Errorf("keyword cannot be empty")

3567 }

3568 if systemWellKnownFlags[kw] {

3569 return fmt.Errorf("cannot use well-known flag as keyword")

3570 }

3571 for _, c := range kw {

3572 // ../rfc/9051:6334

3573 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

3574 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

3575 }

3576 }

3577 return nil

3578}

3579

3580// SendLimitReached checks whether sending a message to recipients would reach

3581// the limit of outgoing messages for the account. If so, the message should

3582// not be sent. If the returned numbers are >= 0, the limit was reached and the

3583// values are the configured limits.

3584//

3585// To limit damage to the internet and our reputation in case of account

3586// compromise, we limit the max number of messages sent in a 24 hour window, both

3587// total number of messages and number of first-time recipients.

3588func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

3589 conf, _ := a.Conf()

3590 msgmax := conf.MaxOutgoingMessagesPerDay

3591 if msgmax == 0 {

3592 // For human senders, 1000 recipients in a day is quite a lot.

3593 msgmax = 1000

3594 }

3595 rcptmax := conf.MaxFirstTimeRecipientsPerDay

3596 if rcptmax == 0 {

3597 // Human senders may address a new human-sized list of people once in a while. In

3598 // case of a compromise, a spammer will probably try to send to many new addresses.

3599 rcptmax = 200

3600 }

3601

3602 rcpts := map[string]time.Time{}

3603 n := 0

3604 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

3605 n++

3606 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

3607 rcpts[o.Recipient] = o.Submitted

3608 }

3609 return nil

3610 })

3611 if err != nil {

3612 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

3613 }

3614 if n+len(recipients) > msgmax {

3615 return msgmax, -1, nil

3616 }

3617

3618 // Only check if max first-time recipients is reached if there are enough messages

3619 // to trigger the limit.

3620 if n+len(recipients) < rcptmax {

3621 return -1, -1, nil

3622 }

3623

3624 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

3625 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

3626 return !exists, err

3627 }

3628

3629 firsttime := 0

3630 now := time.Now()

3631 for _, r := range recipients {

3632 if first, err := isFirstTime(r.XString(true), now); err != nil {

3633 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

3634 } else if first {

3635 firsttime++

3636 }

3637 }

3638 for r, t := range rcpts {

3639 if first, err := isFirstTime(r, t); err != nil {

3640 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

3641 } else if first {

3642 firsttime++

3643 }

3644 }

3645 if firsttime > rcptmax {

3646 return -1, rcptmax, nil

3647 }

3648 return -1, -1, nil

3649}

3650

3651var ErrMailboxExpunged = errors.New("mailbox was deleted")

3652

3653// MailboxID gets a mailbox by ID.

3654//

3655// Returns bstore.ErrAbsent if the mailbox does not exist.

3656// Returns ErrMailboxExpunged if the mailbox is expunged.

3657func MailboxID(tx *bstore.Tx, id int64) (Mailbox, error) {

3658 mb := Mailbox{ID: id}

3659 err := tx.Get(&mb)

3660 if err == nil && mb.Expunged {

3661 return Mailbox{}, ErrMailboxExpunged

3662 }

3663 return mb, err

3664}

3665

3666// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

3667// the total list of created mailboxes is returned in created. On success, if

3668// exists is false and rerr nil, the changes must be broadcasted by the caller.

3669//

3670// The mailbox is created with special-use flags, with those flags taken away from

3671// other mailboxes if they have them, reflected in the returned changes.

3672//

3673// Name must be in normalized form, see config.CheckMailboxName.

3674func (a *Account) MailboxCreate(tx *bstore.Tx, name string, specialUse SpecialUse) (nmb Mailbox, changes []Change, created []string, exists bool, rerr error) {

3675 elems := strings.Split(name, "/")

3676 var p strings.Builder

3677 var modseq ModSeq

3678 for i, elem := range elems {

3679 if i > 0 {

3680 p.WriteString("/")

3681 }

3682 p.WriteString(elem)

3683 exists, err := a.MailboxExists(tx, p.String())

3684 if err != nil {

3685 return Mailbox{}, nil, nil, false, fmt.Errorf("checking if mailbox exists")

3686 }

3687 if exists {

3688 if i == len(elems)-1 {

3689 return Mailbox{}, nil, nil, true, fmt.Errorf("mailbox already exists")

3690 }

3691 continue

3692 }

3693 mb, nchanges, err := a.MailboxEnsure(tx, p.String(), true, specialUse, &modseq)

3694 if err != nil {

3695 return Mailbox{}, nil, nil, false, fmt.Errorf("ensuring mailbox exists: %v", err)

3696 }

3697 nmb = mb

3698 changes = append(changes, nchanges...)

3699 created = append(created, p.String())

3700 }

3701 return nmb, changes, created, false, nil

3702}

3703

3704// MailboxRename renames mailbox mbsrc to dst, including children of mbsrc, and

3705// adds missing parents for dst.

3706//

3707// Name must be in normalized form, see config.CheckMailboxName, and cannot be Inbox.

3708func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc *Mailbox, dst string, modseq *ModSeq) (changes []Change, isInbox, alreadyExists bool, rerr error) {

3709 if mbsrc.Name == "Inbox" || dst == "Inbox" {

3710 return nil, true, false, fmt.Errorf("inbox cannot be renamed")

3711 }

3712

3713 // Check if destination mailbox already exists.

3714 if exists, err := a.MailboxExists(tx, dst); err != nil {

3715 return nil, false, false, fmt.Errorf("checking if destination mailbox exists: %v", err)

3716 } else if exists {

3717 return nil, false, true, fmt.Errorf("destination mailbox already exists")

3718 }

3719

3720 if *modseq == 0 {

3721 var err error

3722 *modseq, err = a.NextModSeq(tx)

3723 if err != nil {

3724 return nil, false, false, fmt.Errorf("get next modseq: %v", err)

3725 }

3726 }

3727

3728 origName := mbsrc.Name

3729

3730 // Move children to their new name.

3731 srcPrefix := mbsrc.Name + "/"

3732 q := bstore.QueryTx[Mailbox](tx)

3733 q.FilterEqual("Expunged", false)

3734 q.FilterFn(func(mb Mailbox) bool {

3735 return strings.HasPrefix(mb.Name, srcPrefix)

3736 })

3737 q.SortDesc("Name") // From leaf towards dst.

3738 kids, err := q.List()

3739 if err != nil {

3740 return nil, false, false, fmt.Errorf("listing child mailboxes")

3741 }

3742

3743 // Rename children, from leaf towards dst (because sorted reverse by name).

3744 for _, mb := range kids {

3745 nname := dst + "/" + mb.Name[len(mbsrc.Name)+1:]

3746 var flags []string

3747 if err := tx.Get(&Subscription{nname}); err == nil {

3748 flags = []string{`\Subscribed`}

3749 } else if err != bstore.ErrAbsent {

3750 return nil, false, false, fmt.Errorf("look up subscription for new name of child %q: %v", nname, err)

3751 }

3752 // Leaf is first.

3753 changes = append(changes, ChangeRenameMailbox{mb.ID, mb.Name, nname, flags, *modseq})

3754

3755 mb.Name = nname

3756 mb.ModSeq = *modseq

3757 if err := tx.Update(&mb); err != nil {

3758 return nil, false, false, fmt.Errorf("rename child mailbox %q: %v", mb.Name, err)

3759 }

3760 }

3761

3762 // Move name out of the way. We may have to create it again, as our new parent.

3763 var flags []string

3764 if err := tx.Get(&Subscription{dst}); err == nil {

3765 flags = []string{`\Subscribed`}

3766 } else if err != bstore.ErrAbsent {

3767 return nil, false, false, fmt.Errorf("look up subscription for new name %q: %v", dst, err)

3768 }

3769 changes = append(changes, ChangeRenameMailbox{mbsrc.ID, mbsrc.Name, dst, flags, *modseq})

3770 mbsrc.ModSeq = *modseq

3771 mbsrc.Name = dst

3772 if err := tx.Update(mbsrc); err != nil {

3773 return nil, false, false, fmt.Errorf("rename mailbox: %v", err)

3774 }

3775

3776 // Add any missing parents for the new name. A mailbox may have been renamed from

3777 // a/b to a/b/x/y, and we'll have to add a new "a" and a/b.

3778 t := strings.Split(dst, "/")

3779 t = t[:len(t)-1]

3780 var parent Mailbox

3781 var parentChanges []Change

3782 for i := range t {

3783 s := strings.Join(t[:i+1], "/")

3784 q := bstore.QueryTx[Mailbox](tx)

3785 q.FilterEqual("Expunged", false)

3786 q.FilterNonzero(Mailbox{Name: s})

3787 pmb, err := q.Get()

3788 if err == nil {

3789 parent = pmb

3790 continue

3791 } else if err != bstore.ErrAbsent {

3792 return nil, false, false, fmt.Errorf("lookup destination parent mailbox %q: %v", s, err)

3793 }

3794

3795 uidval, err := a.NextUIDValidity(tx)

3796 if err != nil {

3797 return nil, false, false, fmt.Errorf("next uid validity: %v", err)

3798 }

3799 parent = Mailbox{

3800 CreateSeq: *modseq,

3801 ModSeq: *modseq,

3802 ParentID: parent.ID,

3803 Name: s,

3804 UIDValidity: uidval,

3805 UIDNext: 1,

3806 HaveCounts: true,

3807 }

3808 if err := tx.Insert(&parent); err != nil {

3809 return nil, false, false, fmt.Errorf("inserting destination parent mailbox %q: %v", s, err)

3810 }

3811

3812 var flags []string

3813 if err := tx.Get(&Subscription{parent.Name}); err == nil {

3814 flags = []string{`\Subscribed`}

3815 } else if err != bstore.ErrAbsent {

3816 return nil, false, false, fmt.Errorf("look up subscription for new parent %q: %v", parent.Name, err)

3817 }

3818 parentChanges = append(parentChanges, ChangeAddMailbox{parent, flags})

3819 }

3820

3821 mbsrc.ParentID = parent.ID

3822 if err := tx.Update(mbsrc); err != nil {

3823 return nil, false, false, fmt.Errorf("set parent id on rename mailbox: %v", err)

3824 }

3825

3826 // If we were moved from a/b to a/b/x, we mention the creation of a/b after we mentioned the rename.

3827 if strings.HasPrefix(dst, origName+"/") {

3828 changes = append(changes, parentChanges...)

3829 } else {

3830 changes = slices.Concat(parentChanges, changes)

3831 }

3832

3833 return changes, false, false, nil

3834}

3835

3836// MailboxDelete marks a mailbox as deleted, including its annotations. If it has

3837// children, the return value indicates that and an error is returned.

3838//

3839// Caller should broadcast the changes (deleting all messages in the mailbox and

3840// deleting the mailbox itself).

3841func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox) (changes []Change, hasChildren bool, rerr error) {

3842 // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about

3843 // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.

3844 qmb := bstore.QueryTx[Mailbox](tx)

3845 qmb.FilterEqual("Expunged", false)

3846 mbprefix := mb.Name + "/"

3847 qmb.FilterFn(func(xmb Mailbox) bool {

3848 return strings.HasPrefix(xmb.Name, mbprefix)

3849 })

3850 if childExists, err := qmb.Exists(); err != nil {

3851 return nil, false, fmt.Errorf("checking if mailbox has child: %v", err)

3852 } else if childExists {

3853 return nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")

3854 }

3855

3856 modseq, err := a.NextModSeq(tx)

3857 if err != nil {

3858 return nil, false, fmt.Errorf("get next modseq: %v", err)

3859 }

3860

3861 qm := bstore.QueryTx[Message](tx)

3862 qm.FilterNonzero(Message{MailboxID: mb.ID})

3863 qm.FilterEqual("Expunged", false)

3864 qm.SortAsc("UID")

3865 l, err := qm.List()

3866 if err != nil {

3867 return nil, false, fmt.Errorf("listing messages in mailbox to remove; %v", err)

3868 }

3869

3870 if len(l) > 0 {

3871 chrem, _, err := a.MessageRemove(log, tx, modseq, mb, RemoveOpts{}, l...)

3872 if err != nil {

3873 return nil, false, fmt.Errorf("marking messages removed: %v", err)

3874 }

3875 changes = append(changes, chrem)

3876 }

3877

3878 // Marking metadata annotations deleted. ../rfc/5464:373

3879 qa := bstore.QueryTx[Annotation](tx)

3880 qa.FilterNonzero(Annotation{MailboxID: mb.ID})

3881 qa.FilterEqual("Expunged", false)

3882 if _, err := qa.UpdateFields(map[string]any{"ModSeq": modseq, "Expunged": true, "IsString": false, "Value": []byte(nil)}); err != nil {

3883 return nil, false, fmt.Errorf("removing annotations for mailbox: %v", err)

3884 }

3885 // Not sending changes about annotations on this mailbox, since the entire mailbox

3886 // is being removed.

3887

3888 mb.ModSeq = modseq

3889 mb.Expunged = true

3890 mb.SpecialUse = SpecialUse{}

3891

3892 if err := tx.Update(mb); err != nil {

3893 return nil, false, fmt.Errorf("updating mailbox: %v", err)

3894 }

3895

3896 changes = append(changes, mb.ChangeRemoveMailbox())

3897 return changes, false, nil

3898}

3899