1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

5

6Layout of storage for accounts:

7

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

10

11Index.db holds tables for user information, mailboxes, and messages. Messages

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

21

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

23

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "os"

37 "path/filepath"

38 "runtime/debug"

39 "sort"

40 "strconv"

41 "strings"

42 "sync"

43 "time"

44

45 "golang.org/x/crypto/bcrypt"

46 "golang.org/x/exp/slices"

47 "golang.org/x/text/unicode/norm"

48

49 "github.com/mjl-/bstore"

50

51 "github.com/mjl-/mox/config"

52 "github.com/mjl-/mox/dns"

53 "github.com/mjl-/mox/message"

54 "github.com/mjl-/mox/metrics"

55 "github.com/mjl-/mox/mlog"

56 "github.com/mjl-/mox/mox-"

57 "github.com/mjl-/mox/moxio"

58 "github.com/mjl-/mox/moxvar"

59 "github.com/mjl-/mox/publicsuffix"

60 "github.com/mjl-/mox/scram"

61 "github.com/mjl-/mox/smtp"

62)

63

64// If true, each time an account is closed its database file is checked for

65// consistency. If an inconsistency is found, panic is called. Set by default

66// because of all the packages with tests, the mox main function sets it to

67// false again.

68var CheckConsistencyOnClose = true

69

70var xlog = mlog.New("store")

71

72var (

73 ErrUnknownMailbox = errors.New("no such mailbox")

74 ErrUnknownCredentials = errors.New("credentials not found")

75 ErrAccountUnknown = errors.New("no such account")

76)

77

78var DefaultInitialMailboxes = config.InitialMailboxes{

79 SpecialUse: config.SpecialUseMailboxes{

80 Sent: "Sent",

81 Archive: "Archive",

82 Trash: "Trash",

83 Draft: "Drafts",

84 Junk: "Junk",

85 },

86}

87

88type SCRAM struct {

89 Salt []byte

90 Iterations int

91 SaltedPassword []byte

92}

93

94// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

95// block with (a derivation of) the key/password, so we don't store the password in plain

96// text.

97type CRAMMD5 struct {

98 Ipad hash.Hash

99 Opad hash.Hash

100}

101

102// BinaryMarshal is used by bstore to store the ipad/opad hash states.

103func (c CRAMMD5) MarshalBinary() ([]byte, error) {

104 if c.Ipad == nil || c.Opad == nil {

105 return nil, nil

106 }

107

108 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

109 if err != nil {

110 return nil, fmt.Errorf("marshal ipad: %v", err)

111 }

112 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

113 if err != nil {

114 return nil, fmt.Errorf("marshal opad: %v", err)

115 }

116 buf := make([]byte, 2+len(ipad)+len(opad))

117 ipadlen := uint16(len(ipad))

118 buf[0] = byte(ipadlen >> 8)

119 buf[1] = byte(ipadlen >> 0)

120 copy(buf[2:], ipad)

121 copy(buf[2+len(ipad):], opad)

122 return buf, nil

123}

124

125// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

126func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

127 if len(buf) == 0 {

128 *c = CRAMMD5{}

129 return nil

130 }

131 if len(buf) < 2 {

132 return fmt.Errorf("short buffer")

133 }

134 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

135 if len(buf) < 2+ipadlen {

136 return fmt.Errorf("buffer too short for ipadlen")

137 }

138 ipad := md5.New()

139 opad := md5.New()

140 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

141 return fmt.Errorf("unmarshal ipad: %v", err)

142 }

143 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

144 return fmt.Errorf("unmarshal opad: %v", err)

145 }

146 *c = CRAMMD5{ipad, opad}

147 return nil

148}

149

150// Password holds credentials in various forms, for logging in with SMTP/IMAP.

151type Password struct {

152 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

153 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

154 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

155 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

156}

157

158// Subjectpass holds the secret key used to sign subjectpass tokens.

159type Subjectpass struct {

160 Email string // Our destination address (canonical, with catchall localpart stripped).

161 Key string

162}

163

164// NextUIDValidity is a singleton record in the database with the next UIDValidity

165// to use for the next mailbox.

166type NextUIDValidity struct {

167 ID int // Just a single record with ID 1.

168 Next uint32

169}

170

171// SyncState track ModSeqs.

172type SyncState struct {

173 ID int // Just a single record with ID 1.

174

175 // Last used, next assigned will be one higher. The first value we hand out is 2.

176 // That's because 0 (the default value for old existing messages, from before the

177 // Message.ModSeq field) is special in IMAP, so we return it as 1.

178 LastModSeq ModSeq `bstore:"nonzero"`

179

180 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

181 // and requests changes based on a modseq before this one, we don't have the

182 // history to provide information about deletions. We normally keep these expunged

183 // records around, but we may periodically truly delete them to reclaim storage

184 // space. Initially set to -1 because we don't want to match with any ModSeq in the

185 // database, which can be zero values.

186 HighestDeletedModSeq ModSeq

187}

188

189// Mailbox is collection of messages, e.g. Inbox or Sent.

190type Mailbox struct {

191 ID int64

192

193 // "Inbox" is the name for the special IMAP "INBOX". Slash separated

194 // for hierarchy.

195 Name string `bstore:"nonzero,unique"`

196

197 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

198 // name, UIDValidity must be changed. Used by IMAP for synchronization.

199 UIDValidity uint32

200

201 // UID likely to be assigned to next message. Used by IMAP to detect messages

202 // delivered to a mailbox.

203 UIDNext UID

204

205 SpecialUse

206

207 // Keywords as used in messages. Storing a non-system keyword for a message

208 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

209 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

210 // lower case (for JMAP), sorted.

211 Keywords []string

212

213 HaveCounts bool // Whether MailboxCounts have been initialized.

214 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

215}

216

217// MailboxCounts tracks statistics about messages for a mailbox.

218type MailboxCounts struct {

219 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

220 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

221 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

222 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

223 Size int64 // Number of bytes for all messages.

224}

225

226func (mc MailboxCounts) String() string {

227 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

228}

229

230// Add increases mailbox counts mc with those of delta.

231func (mc *MailboxCounts) Add(delta MailboxCounts) {

232 mc.Total += delta.Total

233 mc.Deleted += delta.Deleted

234 mc.Unread += delta.Unread

235 mc.Unseen += delta.Unseen

236 mc.Size += delta.Size

237}

238

239// Add decreases mailbox counts mc with those of delta.

240func (mc *MailboxCounts) Sub(delta MailboxCounts) {

241 mc.Total -= delta.Total

242 mc.Deleted -= delta.Deleted

243 mc.Unread -= delta.Unread

244 mc.Unseen -= delta.Unseen

245 mc.Size -= delta.Size

246}

247

248// SpecialUse identifies a specific role for a mailbox, used by clients to

249// understand where messages should go.

250type SpecialUse struct {

251 Archive bool

252 Draft bool

253 Junk bool

254 Sent bool

255 Trash bool

256}

257

258// CalculateCounts calculates the full current counts for messages in the mailbox.

259func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

260 q := bstore.QueryTx[Message](tx)

261 q.FilterNonzero(Message{MailboxID: mb.ID})

262 q.FilterEqual("Expunged", false)

263 err = q.ForEach(func(m Message) error {

264 mc.Add(m.MailboxCounts())

265 return nil

266 })

267 return

268}

269

270// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

271// other connections.

272func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

273 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse}

274}

275

276// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

277// setting a new keyword on a message in the mailbox), for broadcasting to other

278// connections.

279func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

280 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

281}

282

283// KeywordsChanged returns whether the keywords in a mailbox have changed.

284func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

285 if len(mb.Keywords) != len(origmb.Keywords) {

286 return true

287 }

288 // Keywords are stored sorted.

289 for i, kw := range mb.Keywords {

290 if origmb.Keywords[i] != kw {

291 return true

292 }

293 }

294 return false

295}

296

297// CountsChange returns a change with mailbox counts.

298func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

299 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

300}

301

302// Subscriptions are separate from existence of mailboxes.

303type Subscription struct {

304 Name string

305}

306

307// Flags for a mail message.

308type Flags struct {

309 Seen bool

310 Answered bool

311 Flagged bool

312 Forwarded bool

313 Junk bool

314 Notjunk bool

315 Deleted bool

316 Draft bool

317 Phishing bool

318 MDNSent bool

319}

320

321// FlagsAll is all flags set, for use as mask.

322var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

323

324// Validation of "message From" domain.

325type Validation uint8

326

327const (

328 ValidationUnknown Validation = 0

329 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

330 ValidationDMARC Validation = 2 // Actual DMARC policy.

331 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

332 ValidationPass Validation = 4 // For SPF.

333 ValidationNeutral Validation = 5 // For SPF.

334 ValidationTemperror Validation = 6

335 ValidationPermerror Validation = 7

336 ValidationFail Validation = 8

337 ValidationSoftfail Validation = 9 // For SPF.

338 ValidationNone Validation = 10 // E.g. No records.

339)

340

341// Message stored in database and per-message file on disk.

342//

343// Contents are always the combined data from MsgPrefix and the on-disk file named

344// based on ID.

345//

346// Messages always have a header section, even if empty. Incoming messages without

347// header section must get an empty header section added before inserting.

348type Message struct {

349 // ID, unchanged over lifetime, determines path to on-disk msg file.

350 // Set during deliver.

351 ID int64

352

353 UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.

354 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

355

356 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

357 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

358 // always <= ModSeq. If Expunged is set, the message has been removed and should not

359 // be returned to the user. In this case, ModSeq is the Seq where the message is

360 // removed, and will never be changed again.

361 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

362 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

363 // The index on CreateSeq helps efficiently finding created messages for JMAP.

364 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

365 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

366 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

367 // return modseq to clients, we turn 0 into 1.

368 ModSeq ModSeq `bstore:"index"`

369 CreateSeq ModSeq `bstore:"index"`

370 Expunged bool

371

372 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

373 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

374 // flag cleared.

375 IsReject bool

376

377 // If set, this is a forwarded message (through a ruleset with IsForward). This

378 // causes fields used during junk analysis to be moved to their Orig variants, and

379 // masked IP fields cleared, so they aren't used in junk classifications for

380 // incoming messages. This ensures the forwarded messages don't cause negative

381 // reputation for the forwarding mail server, which may also be sending regular

382 // messages.

383 IsForward bool

384

385 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

386 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

387 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

388 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

389 // per-mailbox reputation.

390 //

391 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

392 // mailbox, it is set to the intended mailbox according to delivery rules,

393 // typically that of Inbox. When such a message is moved out of Rejects, the

394 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

395 // message is used for reputation calculation for future deliveries to that

396 // mailbox.

397 //

398 // These are not bstore references to prevent having to update all messages in a

399 // mailbox when the original mailbox is removed. Use of these fields requires

400 // checking if the mailbox still exists.

401 MailboxOrigID int64

402 MailboxDestinedID int64

403

404 Received time.Time `bstore:"default now,index"`

405

406 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

407 // masked IPs are used to classify incoming messages. They are left empty for

408 // messages matching a ruleset for forwarded messages.

409 RemoteIP string

410 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

411 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

412 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

413

414 // Only set if present and not an IP address. Unicode string. Empty for forwarded

415 // messages.

416 EHLODomain string `bstore:"index EHLODomain+Received"`

417 MailFrom string // With localpart and domain. Can be empty.

418 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

419 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

420 // messages, but see OrigMailFromDomain.

421 MailFromDomain string `bstore:"index MailFromDomain+Received"`

422 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

423 RcptToDomain string // Unicode string.

424

425 // Parsed "From" message header, used for reputation along with domain validation.

426 MsgFromLocalpart smtp.Localpart

427 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

428 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

429

430 // Simplified statements of the Validation fields below, used for incoming messages

431 // to check reputation.

432 EHLOValidated bool

433 MailFromValidated bool

434 MsgFromValidated bool

435

436 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

437 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

438 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

439

440 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

441 // DKIM domain that matched a ruleset's verified domain is left out, but included

442 // in OrigDKIMDomains.

443 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

444

445 // For forwarded messages,

446 OrigEHLODomain string

447 OrigDKIMDomains []string

448

449 // Canonicalized Message-Id, always lower-case and normalized quoting, without

450 // <>'s. Empty if missing. Used for matching message threads, and to prevent

451 // duplicate reject delivery.

452 MessageID string `bstore:"index"`

453 // lower-case: ../rfc/5256:495

454

455 // For matching threads in case there is no References/In-Reply-To header. It is

456 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

457 SubjectBase string `bstore:"index"`

458 // ../rfc/5256:90

459

460 // Hash of message. For rejects delivery in case there is no Message-ID, only set

461 // when delivered as reject.

462 MessageHash []byte

463

464 // ID of message starting this thread.

465 ThreadID int64 `bstore:"index"`

466 // IDs of parent messages, from closest parent to the root message. Parent messages

467 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

468 // never contain the message id itself (a cycle), and parent messages must

469 // reference the same ancestors.

470 ThreadParentIDs []int64

471 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

472 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

473 // have been deleted), or subject-based matching was done.

474 ThreadMissingLink bool

475 // If set, newly delivered child messages are automatically marked as read. This

476 // field is copied to new child messages. Changes are propagated to the webmail

477 // client.

478 ThreadMuted bool

479 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

480 // "on" (mode "unread" ignores it). This field is copied to new child message.

481 // Changes are propagated to the webmail client.

482 ThreadCollapsed bool

483

484 // If received message was known to match a mailing list rule (with modified junk

485 // filtering).

486 IsMailingList bool

487

488 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

489 ReceivedTLSCipherSuite uint16

490 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

491

492 Flags

493 // For keywords other than system flags or the basic well-known $-flags. Only in

494 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

495 // (for JMAP), sorted.

496 Keywords []string `bstore:"index"`

497 Size int64

498 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

499 MsgPrefix []byte // Typically holds received headers and/or header separator.

500

501 // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore

502 // cannot yet store recursive types. Created when first needed, and saved in the

503 // database.

504 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

505 ParsedBuf []byte

506}

507

508// MailboxCounts returns the delta to counts this message means for its

509// mailbox.

510func (m Message) MailboxCounts() (mc MailboxCounts) {

511 if m.Expunged {

512 return

513 }

514 if m.Deleted {

515 mc.Deleted++

516 } else {

517 mc.Total++

518 }

519 if !m.Seen {

520 mc.Unseen++

521 if !m.Deleted {

522 mc.Unread++

523 }

524 }

525 mc.Size += m.Size

526 return

527}

528

529func (m Message) ChangeAddUID() ChangeAddUID {

530 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}

531}

532

533func (m Message) ChangeFlags(orig Flags) ChangeFlags {

534 mask := m.Flags.Changed(orig)

535 return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}

536}

537

538func (m Message) ChangeThread() ChangeThread {

539 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

540}

541

542// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

543// database is sent to the client as 1, because modseq 0 is special in IMAP.

544// ModSeq coming from the client are of type int64.

545type ModSeq int64

546

547func (ms ModSeq) Client() int64 {

548 if ms == 0 {

549 return 1

550 }

551 return int64(ms)

552}

553

554// ModSeqFromClient converts a modseq from a client to a modseq for internal

555// use, e.g. in a database query.

556// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

557func ModSeqFromClient(modseq int64) ModSeq {

558 if modseq == 1 {

559 return 0

560 }

561 return ModSeq(modseq)

562}

563

564// PrepareExpunge clears fields that are no longer needed after an expunge, so

565// almost all fields. Does not change ModSeq, but does set Expunged.

566func (m *Message) PrepareExpunge() {

567 *m = Message{

568 ID: m.ID,

569 UID: m.UID,

570 MailboxID: m.MailboxID,

571 CreateSeq: m.CreateSeq,

572 ModSeq: m.ModSeq,

573 Expunged: true,

574 ThreadID: m.ThreadID,

575 }

576}

577

578// PrepareThreading sets MessageID and SubjectBase (used in threading) based on the

579// envelope in part.

580func (m *Message) PrepareThreading(log *mlog.Log, part *message.Part) {

581 if part.Envelope == nil {

582 return

583 }

584 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

585 if err != nil {

586 log.Debugx("parsing message-id, ignoring", err, mlog.Field("messageid", part.Envelope.MessageID))

587 } else if raw {

588 log.Debug("could not parse message-id as address, continuing with raw value", mlog.Field("messageid", part.Envelope.MessageID))

589 }

590 m.MessageID = messageID

591 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

592}

593

594// LoadPart returns a message.Part by reading from m.ParsedBuf.

595func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

596 if m.ParsedBuf == nil {

597 return message.Part{}, fmt.Errorf("message not parsed")

598 }

599 var p message.Part

600 err := json.Unmarshal(m.ParsedBuf, &p)

601 if err != nil {

602 return p, fmt.Errorf("unmarshal message part")

603 }

604 p.SetReaderAt(r)

605 return p, nil

606}

607

608// NeedsTraining returns whether message needs a training update, based on

609// TrainedJunk (current training status) and new Junk/Notjunk flags.

610func (m Message) NeedsTraining() bool {

611 untrain := m.TrainedJunk != nil

612 untrainJunk := untrain && *m.TrainedJunk

613 train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)

614 trainJunk := m.Junk

615 return untrain != train || untrain && train && untrainJunk != trainJunk

616}

617

618// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

619// used when delivering/moving/copying messages to a mailbox. Mail clients are not

620// very helpful with setting junk/notjunk flags. But clients can move/copy messages

621// to other mailboxes. So we set flags when clients move a message.

622func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

623 if mb.Junk {

624 m.Junk = true

625 m.Notjunk = false

626 return

627 }

628

629 if !conf.AutomaticJunkFlags.Enabled {

630 return

631 }

632

633 lmailbox := strings.ToLower(mb.Name)

634

635 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

636 m.Junk = true

637 m.Notjunk = false

638 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

639 m.Junk = false

640 m.Notjunk = false

641 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

642 m.Junk = false

643 m.Notjunk = true

644 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

645 m.Junk = true

646 m.Notjunk = false

647 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

648 m.Junk = false

649 m.Notjunk = false

650 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

651 m.Junk = false

652 m.Notjunk = true

653 }

654}

655

656// Recipient represents the recipient of a message. It is tracked to allow

657// first-time incoming replies from users this account has sent messages to. When a

658// mailbox is added to the Sent mailbox the message is parsed and recipients are

659// inserted as recipient. Recipients are never removed other than for removing the

660// message. On move/copy of a message, recipients aren't modified either. For IMAP,

661// this assumes a client simply appends messages to the Sent mailbox (as opposed to

662// copying messages from some place).

663type Recipient struct {

664 ID int64

665 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

666 Localpart smtp.Localpart `bstore:"nonzero"`

667 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

668 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

669 Sent time.Time `bstore:"nonzero"`

670}

671

672// Outgoing is a message submitted for delivery from the queue. Used to enforce

673// maximum outgoing messages.

674type Outgoing struct {

675 ID int64

676 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

677 Submitted time.Time `bstore:"nonzero,default now"`

678}

679

680// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

681// during most recent connection (delivery attempt).

682type RecipientDomainTLS struct {

683 Domain string // Unicode.

684 Updated time.Time `bstore:"default now"`

685 STARTTLS bool // Supports STARTTLS.

686 RequireTLS bool // Supports RequireTLS SMTP extension.

687}

688

689// Types stored in DB.

690var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}, Upgrade{}, RecipientDomainTLS{}}

691

692// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

693type Account struct {

694 Name string // Name, according to configuration.

695 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

696 DBPath string // Path to database with mailboxes, messages, etc.

697 DB *bstore.DB // Open database connection.

698

699 // Channel that is closed if/when account has/gets "threads" accounting (see

700 // Upgrade.Threads).

701 threadsCompleted chan struct{}

702 // If threads upgrade completed with error, this is set. Used for warning during

703 // delivery, or aborting when importing.

704 threadsErr error

705

706 // Write lock must be held for account/mailbox modifications including message delivery.

707 // Read lock for reading mailboxes/messages.

708 // When making changes to mailboxes/messages, changes must be broadcasted before

709 // releasing the lock to ensure proper UID ordering.

710 sync.RWMutex

711

712 nused int // Reference count, while >0, this account is alive and shared.

713}

714

715type Upgrade struct {

716 ID byte

717 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

718}

719

720// InitialUIDValidity returns a UIDValidity used for initializing an account.

721// It can be replaced during tests with a predictable value.

722var InitialUIDValidity = func() uint32 {

723 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

724}

725

726var openAccounts = struct {

727 names map[string]*Account

728 sync.Mutex

729}{

730 names: map[string]*Account{},

731}

732

733func closeAccount(acc *Account) (rerr error) {

734 openAccounts.Lock()

735 acc.nused--

736 defer openAccounts.Unlock()

737 if acc.nused == 0 {

738 // threadsCompleted must be closed now because it increased nused.

739 rerr = acc.DB.Close()

740 acc.DB = nil

741 delete(openAccounts.names, acc.Name)

742 }

743 return

744}

745

746// OpenAccount opens an account by name.

747//

748// No additional data path prefix or ".db" suffix should be added to the name.

749// A single shared account exists per name.

750func OpenAccount(name string) (*Account, error) {

751 openAccounts.Lock()

752 defer openAccounts.Unlock()

753 if acc, ok := openAccounts.names[name]; ok {

754 acc.nused++

755 return acc, nil

756 }

757

758 if _, ok := mox.Conf.Account(name); !ok {

759 return nil, ErrAccountUnknown

760 }

761

762 acc, err := openAccount(name)

763 if err != nil {

764 return nil, err

765 }

766 openAccounts.names[name] = acc

767 return acc, nil

768}

769

770// openAccount opens an existing account, or creates it if it is missing.

771func openAccount(name string) (a *Account, rerr error) {

772 dir := filepath.Join(mox.DataDirPath("accounts"), name)

773 return OpenAccountDB(dir, name)

774}

775

776// OpenAccountDB opens an account database file and returns an initialized account

777// or error. Only exported for use by subcommands that verify the database file.

778// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

779func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {

780 dbpath := filepath.Join(accountDir, "index.db")

781

782 // Create account if it doesn't exist yet.

783 isNew := false

784 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

785 isNew = true

786 os.MkdirAll(accountDir, 0770)

787 }

788

789 db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)

790 if err != nil {

791 return nil, err

792 }

793

794 defer func() {

795 if rerr != nil {

796 db.Close()

797 if isNew {

798 os.Remove(dbpath)

799 }

800 }

801 }()

802

803 acc := &Account{

804 Name: accountName,

805 Dir: accountDir,

806 DBPath: dbpath,

807 DB: db,

808 nused: 1,

809 threadsCompleted: make(chan struct{}),

810 }

811

812 if isNew {

813 if err := initAccount(db); err != nil {

814 return nil, fmt.Errorf("initializing account: %v", err)

815 }

816 close(acc.threadsCompleted)

817 return acc, nil

818 }

819

820 // Ensure mailbox counts are set.

821 var mentioned bool

822 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

823 return bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

824 if !mentioned {

825 mentioned = true

826 xlog.Info("first calculation of mailbox counts for account", mlog.Field("account", accountName))

827 }

828 mc, err := mb.CalculateCounts(tx)

829 if err != nil {

830 return err

831 }

832 mb.HaveCounts = true

833 mb.MailboxCounts = mc

834 return tx.Update(&mb)

835 })

836 })

837 if err != nil {

838 return nil, fmt.Errorf("calculating counts for mailbox: %v", err)

839 }

840

841 // Start adding threading if needed.

842 up := Upgrade{ID: 1}

843 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

844 err := tx.Get(&up)

845 if err == bstore.ErrAbsent {

846 if err := tx.Insert(&up); err != nil {

847 return fmt.Errorf("inserting initial upgrade record: %v", err)

848 }

849 err = nil

850 }

851 return err

852 })

853 if err != nil {

854 return nil, fmt.Errorf("checking message threading: %v", err)

855 }

856 if up.Threads == 2 {

857 close(acc.threadsCompleted)

858 return acc, nil

859 }

860

861 // Increase account use before holding on to account in background.

862 // Caller holds the lock. The goroutine below decreases nused by calling

863 // closeAccount.

864 acc.nused++

865

866 // Ensure all messages have a MessageID and SubjectBase, which are needed when

867 // matching threads.

868 // Then assign messages to threads, in the same way we do during imports.

869 xlog.Info("upgrading account for threading, in background", mlog.Field("account", acc.Name))

870 go func() {

871 defer func() {

872 err := closeAccount(acc)

873 xlog.Check(err, "closing use of account after upgrading account storage for threads", mlog.Field("account", a.Name))

874 }()

875

876 defer func() {

877 x := recover() // Should not happen, but don't take program down if it does.

878 if x != nil {

879 xlog.Error("upgradeThreads panic", mlog.Field("err", x))

880 debug.PrintStack()

881 metrics.PanicInc(metrics.Upgradethreads)

882 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

883 }

884

885 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

886 close(acc.threadsCompleted)

887 }()

888

889 err := upgradeThreads(mox.Shutdown, acc, &up)

890 if err != nil {

891 a.threadsErr = err

892 xlog.Errorx("upgrading account for threading, aborted", err, mlog.Field("account", a.Name))

893 } else {

894 xlog.Info("upgrading account for threading, completed", mlog.Field("account", a.Name))

895 }

896 }()

897 return acc, nil

898}

899

900// ThreadingWait blocks until the one-time account threading upgrade for the

901// account has completed, and returns an error if not successful.

902//

903// To be used before starting an import of messages.

904func (a *Account) ThreadingWait(log *mlog.Log) error {

905 select {

906 case <-a.threadsCompleted:

907 return a.threadsErr

908 default:

909 }

910 log.Debug("waiting for account upgrade to complete")

911

912 <-a.threadsCompleted

913 return a.threadsErr

914}

915

916func initAccount(db *bstore.DB) error {

917 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

918 uidvalidity := InitialUIDValidity()

919

920 if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {

921 return err

922 }

923

924 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

925 // Deprecated in favor of InitialMailboxes.

926 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

927 mailboxes := []string{"Inbox"}

928 for _, name := range defaultMailboxes {

929 if strings.EqualFold(name, "Inbox") {

930 continue

931 }

932 mailboxes = append(mailboxes, name)

933 }

934 for _, name := range mailboxes {

935 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, HaveCounts: true}

936 if strings.HasPrefix(name, "Archive") {

937 mb.Archive = true

938 } else if strings.HasPrefix(name, "Drafts") {

939 mb.Draft = true

940 } else if strings.HasPrefix(name, "Junk") {

941 mb.Junk = true

942 } else if strings.HasPrefix(name, "Sent") {

943 mb.Sent = true

944 } else if strings.HasPrefix(name, "Trash") {

945 mb.Trash = true

946 }

947 if err := tx.Insert(&mb); err != nil {

948 return fmt.Errorf("creating mailbox: %w", err)

949 }

950 if err := tx.Insert(&Subscription{name}); err != nil {

951 return fmt.Errorf("adding subscription: %w", err)

952 }

953 }

954 } else {

955 mailboxes := mox.Conf.Static.InitialMailboxes

956 var zerouse config.SpecialUseMailboxes

957 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

958 mailboxes = DefaultInitialMailboxes

959 }

960

961 add := func(name string, use SpecialUse) error {

962 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, HaveCounts: true}

963 if err := tx.Insert(&mb); err != nil {

964 return fmt.Errorf("creating mailbox: %w", err)

965 }

966 if err := tx.Insert(&Subscription{name}); err != nil {

967 return fmt.Errorf("adding subscription: %w", err)

968 }

969 return nil

970 }

971 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

972 if nameOpt == "" {

973 return nil

974 }

975 return add(nameOpt, use)

976 }

977 l := []struct {

978 nameOpt string

979 use SpecialUse

980 }{

981 {"Inbox", SpecialUse{}},

982 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

983 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

984 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

985 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

986 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

987 }

988 for _, e := range l {

989 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

990 return err

991 }

992 }

993 for _, name := range mailboxes.Regular {

994 if err := add(name, SpecialUse{}); err != nil {

995 return err

996 }

997 }

998 }

999

1000 uidvalidity++

1001 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1002 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1003 }

1004 return nil

1005 })

1006}

1007

1008// Close reduces the reference count, and closes the database connection when

1009// it was the last user.

1010func (a *Account) Close() error {

1011 if CheckConsistencyOnClose {

1012 xerr := a.CheckConsistency()

1013 err := closeAccount(a)

1014 if xerr != nil {

1015 panic(xerr)

1016 }

1017 return err

1018 }

1019 return closeAccount(a)

1020}

1021

1022// CheckConsistency checks the consistency of the database and returns a non-nil

1023// error for these cases:

1024//

1025// - Missing on-disk file for message.

1026// - Mismatch between message size and length of MsgPrefix and on-disk file.

1027// - Missing HaveCounts.

1028// - Incorrect mailbox counts.

1029// - Message with UID >= mailbox uid next.

1030// - Mailbox uidvalidity >= account uid validity.

1031// - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1032// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1033func (a *Account) CheckConsistency() error {

1034 var uidErrors []string // With a limit, could be many.

1035 var modseqErrors []string // With limit.

1036 var fileErrors []string // With limit.

1037 var threadidErrors []string // With limit.

1038 var threadParentErrors []string // With limit.

1039 var threadAncestorErrors []string // With limit.

1040 var errors []string

1041

1042 err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {

1043 nuv := NextUIDValidity{ID: 1}

1044 err := tx.Get(&nuv)

1045 if err != nil {

1046 return fmt.Errorf("fetching next uid validity: %v", err)

1047 }

1048

1049 mailboxes := map[int64]Mailbox{}

1050 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1051 mailboxes[mb.ID] = mb

1052

1053 if mb.UIDValidity >= nuv.Next {

1054 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1055 errors = append(errors, errmsg)

1056 }

1057 return nil

1058 })

1059 if err != nil {

1060 return fmt.Errorf("listing mailboxes: %v", err)

1061 }

1062

1063 counts := map[int64]MailboxCounts{}

1064 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1065 mc := counts[m.MailboxID]

1066 mc.Add(m.MailboxCounts())

1067 counts[m.MailboxID] = mc

1068

1069 mb := mailboxes[m.MailboxID]

1070

1071 if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1072 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1073 modseqErrors = append(modseqErrors, modseqerr)

1074 }

1075 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1076 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1077 uidErrors = append(uidErrors, uiderr)

1078 }

1079 if m.Expunged {

1080 return nil

1081 }

1082 p := a.MessagePath(m.ID)

1083 st, err := os.Stat(p)

1084 if err != nil {

1085 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1086 fileErrors = append(fileErrors, existserr)

1087 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1088 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1089 fileErrors = append(fileErrors, sizeerr)

1090 }

1091

1092 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1093 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1094 threadidErrors = append(threadidErrors, err)

1095 }

1096 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1097 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1098 threadParentErrors = append(threadParentErrors, err)

1099 }

1100 for i, pid := range m.ThreadParentIDs {

1101 am := Message{ID: pid}

1102 if err := tx.Get(&am); err == bstore.ErrAbsent {

1103 continue

1104 } else if err != nil {

1105 return fmt.Errorf("get ancestor message: %v", err)

1106 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

1107 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

1108 threadAncestorErrors = append(threadAncestorErrors, err)

1109 } else {

1110 break

1111 }

1112 }

1113 return nil

1114 })

1115 if err != nil {

1116 return fmt.Errorf("reading messages: %v", err)

1117 }

1118

1119 for _, mb := range mailboxes {

1120 if !mb.HaveCounts {

1121 errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID])

1122 errors = append(errors, errmsg)

1123 } else if mb.MailboxCounts != counts[mb.ID] {

1124 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

1125 errors = append(errors, mbcounterr)

1126 }

1127 }

1128

1129 return nil

1130 })

1131 if err != nil {

1132 return err

1133 }

1134 errors = append(errors, uidErrors...)

1135 errors = append(errors, modseqErrors...)

1136 errors = append(errors, fileErrors...)

1137 errors = append(errors, threadidErrors...)

1138 errors = append(errors, threadParentErrors...)

1139 errors = append(errors, threadAncestorErrors...)

1140 if len(errors) > 0 {

1141 return fmt.Errorf("%s", strings.Join(errors, "; "))

1142 }

1143 return nil

1144}

1145

1146// Conf returns the configuration for this account if it still exists. During

1147// an SMTP session, a configuration update may drop an account.

1148func (a *Account) Conf() (config.Account, bool) {

1149 return mox.Conf.Account(a.Name)

1150}

1151

1152// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

1153func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

1154 nuv := NextUIDValidity{ID: 1}

1155 if err := tx.Get(&nuv); err != nil {

1156 return 0, err

1157 }

1158 v := nuv.Next

1159 nuv.Next++

1160 if err := tx.Update(&nuv); err != nil {

1161 return 0, err

1162 }

1163 return v, nil

1164}

1165

1166// NextModSeq returns the next modification sequence, which is global per account,

1167// over all types.

1168func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

1169 v := SyncState{ID: 1}

1170 if err := tx.Get(&v); err == bstore.ErrAbsent {

1171 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

1172 // already used.

1173 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

1174 // makes sense.

1175 v = SyncState{1, 2, -1}

1176 return v.LastModSeq, tx.Insert(&v)

1177 } else if err != nil {

1178 return 0, err

1179 }

1180 v.LastModSeq++

1181 return v.LastModSeq, tx.Update(&v)

1182}

1183

1184func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

1185 v := SyncState{ID: 1}

1186 err := tx.Get(&v)

1187 if err == bstore.ErrAbsent {

1188 return 0, nil

1189 }

1190 return v.HighestDeletedModSeq, err

1191}

1192

1193// WithWLock runs fn with account writelock held. Necessary for account/mailbox modification. For message delivery, a read lock is required.

1194func (a *Account) WithWLock(fn func()) {

1195 a.Lock()

1196 defer a.Unlock()

1197 fn()

1198}

1199

1200// WithRLock runs fn with account read lock held. Needed for message delivery.

1201func (a *Account) WithRLock(fn func()) {

1202 a.RLock()

1203 defer a.RUnlock()

1204 fn()

1205}

1206

1207// DeliverMessage delivers a mail message to the account.

1208//

1209// The message, with msg.MsgPrefix and msgFile combined, must have a header

1210// section. The caller is responsible for adding a header separator to

1211// msg.MsgPrefix if missing from an incoming message.

1212//

1213// If the destination mailbox has the Sent special-use flag, the message is parsed

1214// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

1215// dmarc reputation.

1216//

1217// If sync is true, the message file and its directory are synced. Should be true

1218// for regular mail delivery, but can be false when importing many messages.

1219//

1220// If CreateSeq/ModSeq is not set, it is assigned automatically.

1221//

1222// Must be called with account rlock or wlock.

1223//

1224// Caller must broadcast new message.

1225//

1226// Caller must update mailbox counts.

1227func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads bool) error {

1228 if m.Expunged {

1229 return fmt.Errorf("cannot deliver expunged message")

1230 }

1231

1232 mb := Mailbox{ID: m.MailboxID}

1233 if err := tx.Get(&mb); err != nil {

1234 return fmt.Errorf("get mailbox: %w", err)

1235 }

1236 m.UID = mb.UIDNext

1237 mb.UIDNext++

1238 if err := tx.Update(&mb); err != nil {

1239 return fmt.Errorf("updating mailbox nextuid: %w", err)

1240 }

1241

1242 conf, _ := a.Conf()

1243 m.JunkFlagsForMailbox(mb, conf)

1244

1245 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

1246 var part *message.Part

1247 if m.ParsedBuf == nil {

1248 p, err := message.EnsurePart(log, false, mr, m.Size)

1249 if err != nil {

1250 log.Infox("parsing delivered message", err, mlog.Field("parse", ""), mlog.Field("message", m.ID))

1251 // We continue, p is still valid.

1252 }

1253 part = &p

1254 buf, err := json.Marshal(part)

1255 if err != nil {

1256 return fmt.Errorf("marshal parsed message: %w", err)

1257 }

1258 m.ParsedBuf = buf

1259 } else {

1260 var p message.Part

1261 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

1262 log.Errorx("unmarshal parsed message, continuing", err, mlog.Field("parse", ""))

1263 } else {

1264 part = &p

1265 }

1266 }

1267

1268 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

1269 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

1270 m.MailboxDestinedID = 0

1271 }

1272 if m.CreateSeq == 0 || m.ModSeq == 0 {

1273 modseq, err := a.NextModSeq(tx)

1274 if err != nil {

1275 return fmt.Errorf("assigning next modseq: %w", err)

1276 }

1277 m.CreateSeq = modseq

1278 m.ModSeq = modseq

1279 }

1280

1281 if part != nil && m.MessageID == "" && m.SubjectBase == "" {

1282 m.PrepareThreading(log, part)

1283 }

1284

1285 // Assign to thread (if upgrade has completed).

1286 noThreadID := nothreads

1287 if m.ThreadID == 0 && !nothreads && part != nil {

1288 select {

1289 case <-a.threadsCompleted:

1290 if a.threadsErr != nil {

1291 log.Info("not assigning threads for new delivery, upgrading to threads failed")

1292 noThreadID = true

1293 } else {

1294 if err := assignThread(log, tx, m, part); err != nil {

1295 return fmt.Errorf("assigning thread: %w", err)

1296 }

1297 }

1298 default:

1299 // note: since we have a write transaction to get here, we can't wait for the

1300 // thread upgrade to finish.

1301 // If we don't assign a threadid the upgrade process will do it.

1302 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

1303 noThreadID = true

1304 }

1305 }

1306

1307 if err := tx.Insert(m); err != nil {

1308 return fmt.Errorf("inserting message: %w", err)

1309 }

1310 if !noThreadID && m.ThreadID == 0 {

1311 m.ThreadID = m.ID

1312 if err := tx.Update(m); err != nil {

1313 return fmt.Errorf("updating message for its own thread id: %w", err)

1314 }

1315 }

1316

1317 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's. for webmail, we could insert the recipients directly.

1318 if mb.Sent && part != nil && part.Envelope != nil {

1319 e := part.Envelope

1320 sent := e.Date

1321 if sent.IsZero() {

1322 sent = m.Received

1323 }

1324 if sent.IsZero() {

1325 sent = time.Now()

1326 }

1327 addrs := append(append(e.To, e.CC...), e.BCC...)

1328 for _, addr := range addrs {

1329 if addr.User == "" {

1330 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

1331 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", mlog.Field("address", addr))

1332 continue

1333 }

1334 d, err := dns.ParseDomain(addr.Host)

1335 if err != nil {

1336 log.Debugx("parsing domain in to/cc/bcc address", err, mlog.Field("address", addr))

1337 continue

1338 }

1339 mr := Recipient{

1340 MessageID: m.ID,

1341 Localpart: smtp.Localpart(addr.User),

1342 Domain: d.Name(),

1343 OrgDomain: publicsuffix.Lookup(context.TODO(), d).Name(),

1344 Sent: sent,

1345 }

1346 if err := tx.Insert(&mr); err != nil {

1347 return fmt.Errorf("inserting sent message recipients: %w", err)

1348 }

1349 }

1350 }

1351

1352 msgPath := a.MessagePath(m.ID)

1353 msgDir := filepath.Dir(msgPath)

1354 os.MkdirAll(msgDir, 0770)

1355

1356 // Sync file data to disk.

1357 if sync {

1358 if err := msgFile.Sync(); err != nil {

1359 return fmt.Errorf("fsync message file: %w", err)

1360 }

1361 }

1362

1363 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

1364 return fmt.Errorf("linking/copying message to new file: %w", err)

1365 }

1366

1367 if sync {

1368 if err := moxio.SyncDir(msgDir); err != nil {

1369 xerr := os.Remove(msgPath)

1370 log.Check(xerr, "removing message after syncdir error", mlog.Field("path", msgPath))

1371 return fmt.Errorf("sync directory: %w", err)

1372 }

1373 }

1374

1375 if !notrain && m.NeedsTraining() {

1376 l := []Message{*m}

1377 if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {

1378 xerr := os.Remove(msgPath)

1379 log.Check(xerr, "removing message after syncdir error", mlog.Field("path", msgPath))

1380 return fmt.Errorf("training junkfilter: %w", err)

1381 }

1382 *m = l[0]

1383 }

1384

1385 return nil

1386}

1387

1388// SetPassword saves a new password for this account. This password is used for

1389// IMAP, SMTP (submission) sessions and the HTTP account web page.

1390func (a *Account) SetPassword(password string) error {

1391 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

1392 if err != nil {

1393 return fmt.Errorf("generating password hash: %w", err)

1394 }

1395

1396 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1397 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

1398 return fmt.Errorf("deleting existing password: %v", err)

1399 }

1400 var pw Password

1401 pw.Hash = string(hash)

1402

1403 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

1404 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

1405 // first block is based on the key/password. We hash those first blocks now, and

1406 // store the hash state in the database. When we actually authenticate, we'll

1407 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

1408 // because it does not expose its internal state and isn't a BinaryMarshaler.

1409 // ../rfc/2104:121

1410 pw.CRAMMD5.Ipad = md5.New()

1411 pw.CRAMMD5.Opad = md5.New()

1412 key := []byte(password)

1413 if len(key) > 64 {

1414 t := md5.Sum(key)

1415 key = t[:]

1416 }

1417 ipad := make([]byte, md5.BlockSize)

1418 opad := make([]byte, md5.BlockSize)

1419 copy(ipad, key)

1420 copy(opad, key)

1421 for i := range ipad {

1422 ipad[i] ^= 0x36

1423 opad[i] ^= 0x5c

1424 }

1425 pw.CRAMMD5.Ipad.Write(ipad)

1426 pw.CRAMMD5.Opad.Write(opad)

1427

1428 pw.SCRAMSHA1.Salt = scram.MakeRandom()

1429 pw.SCRAMSHA1.Iterations = 2 * 4096

1430 pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

1431

1432 pw.SCRAMSHA256.Salt = scram.MakeRandom()

1433 pw.SCRAMSHA256.Iterations = 4096

1434 pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

1435

1436 if err := tx.Insert(&pw); err != nil {

1437 return fmt.Errorf("inserting new password: %v", err)

1438 }

1439 return nil

1440 })

1441 if err == nil {

1442 xlog.Info("new password set for account", mlog.Field("account", a.Name))

1443 }

1444 return err

1445}

1446

1447// Subjectpass returns the signing key for use with subjectpass for the given

1448// email address with canonical localpart.

1449func (a *Account) Subjectpass(email string) (key string, err error) {

1450 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1451 v := Subjectpass{Email: email}

1452 err := tx.Get(&v)

1453 if err == nil {

1454 key = v.Key

1455 return nil

1456 }

1457 if !errors.Is(err, bstore.ErrAbsent) {

1458 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

1459 }

1460 key = ""

1461 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

1462 buf := make([]byte, 16)

1463 if _, err := cryptorand.Read(buf); err != nil {

1464 return err

1465 }

1466 for _, b := range buf {

1467 key += string(chars[int(b)%len(chars)])

1468 }

1469 v.Key = key

1470 return tx.Insert(&v)

1471 })

1472}

1473

1474// Ensure mailbox is present in database, adding records for the mailbox and its

1475// parents if they aren't present.

1476//

1477// If subscribe is true, any mailboxes that were created will also be subscribed to.

1478// Caller must hold account wlock.

1479// Caller must propagate changes if any.

1480func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb Mailbox, changes []Change, rerr error) {

1481 if norm.NFC.String(name) != name {

1482 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

1483 }

1484

1485 // Quick sanity check.

1486 if strings.EqualFold(name, "inbox") && name != "Inbox" {

1487 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

1488 }

1489

1490 elems := strings.Split(name, "/")

1491 q := bstore.QueryTx[Mailbox](tx)

1492 q.FilterFn(func(mb Mailbox) bool {

1493 return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/")

1494 })

1495 l, err := q.List()

1496 if err != nil {

1497 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

1498 }

1499

1500 mailboxes := map[string]Mailbox{}

1501 for _, xmb := range l {

1502 mailboxes[xmb.Name] = xmb

1503 }

1504

1505 p := ""

1506 for _, elem := range elems {

1507 if p != "" {

1508 p += "/"

1509 }

1510 p += elem

1511 var ok bool

1512 mb, ok = mailboxes[p]

1513 if ok {

1514 continue

1515 }

1516 uidval, err := a.NextUIDValidity(tx)

1517 if err != nil {

1518 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

1519 }

1520 mb = Mailbox{

1521 Name: p,

1522 UIDValidity: uidval,

1523 UIDNext: 1,

1524 HaveCounts: true,

1525 }

1526 err = tx.Insert(&mb)

1527 if err != nil {

1528 return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err)

1529 }

1530

1531 var flags []string

1532 if subscribe {

1533 if tx.Get(&Subscription{p}) != nil {

1534 err := tx.Insert(&Subscription{p})

1535 if err != nil {

1536 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err)

1537 }

1538 }

1539 flags = []string{`\Subscribed`}

1540 }

1541 changes = append(changes, ChangeAddMailbox{mb, flags})

1542 }

1543 return mb, changes, nil

1544}

1545

1546// MailboxExists checks if mailbox exists.

1547// Caller must hold account rlock.

1548func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

1549 q := bstore.QueryTx[Mailbox](tx)

1550 q.FilterEqual("Name", name)

1551 return q.Exists()

1552}

1553

1554// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

1555func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

1556 q := bstore.QueryTx[Mailbox](tx)

1557 q.FilterEqual("Name", name)

1558 mb, err := q.Get()

1559 if err == bstore.ErrAbsent {

1560 return nil, nil

1561 }

1562 if err != nil {

1563 return nil, fmt.Errorf("looking up mailbox: %w", err)

1564 }

1565 return &mb, nil

1566}

1567

1568// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

1569// have to exist. Any parents are not automatically subscribed.

1570// Changes are returned and must be broadcasted by the caller.

1571func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

1572 if err := tx.Get(&Subscription{name}); err == nil {

1573 return nil, nil

1574 }

1575

1576 if err := tx.Insert(&Subscription{name}); err != nil {

1577 return nil, fmt.Errorf("inserting subscription: %w", err)

1578 }

1579

1580 q := bstore.QueryTx[Mailbox](tx)

1581 q.FilterEqual("Name", name)

1582 _, err := q.Get()

1583 if err == nil {

1584 return []Change{ChangeAddSubscription{name, nil}}, nil

1585 } else if err != bstore.ErrAbsent {

1586 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

1587 }

1588 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

1589}

1590

1591// MessageRuleset returns the first ruleset (if any) that message the message

1592// represented by msgPrefix and msgFile, with smtp and validation fields from m.

1593func MessageRuleset(log *mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

1594 if len(dest.Rulesets) == 0 {

1595 return nil

1596 }

1597

1598 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

1599 p, err := message.Parse(log, false, mr)

1600 if err != nil {

1601 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, mlog.Field("parse", ""))

1602 // note: part is still set.

1603 }

1604 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

1605 header, err := p.Header()

1606 if err != nil {

1607 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, mlog.Field("parse", ""))

1608 // todo: reject message?

1609 return nil

1610 }

1611

1612ruleset:

1613 for _, rs := range dest.Rulesets {

1614 if rs.SMTPMailFromRegexpCompiled != nil {

1615 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

1616 continue ruleset

1617 }

1618 }

1619

1620 if !rs.VerifiedDNSDomain.IsZero() {

1621 d := rs.VerifiedDNSDomain.Name()

1622 suffix := "." + d

1623 matchDomain := func(s string) bool {

1624 return s == d || strings.HasSuffix(s, suffix)

1625 }

1626 var ok bool

1627 if m.EHLOValidated && matchDomain(m.EHLODomain) {

1628 ok = true

1629 }

1630 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

1631 ok = true

1632 }

1633 for _, d := range m.DKIMDomains {

1634 if matchDomain(d) {

1635 ok = true

1636 break

1637 }

1638 }

1639 if !ok {

1640 continue ruleset

1641 }

1642 }

1643

1644 header:

1645 for _, t := range rs.HeadersRegexpCompiled {

1646 for k, vl := range header {

1647 k = strings.ToLower(k)

1648 if !t[0].MatchString(k) {

1649 continue

1650 }

1651 for _, v := range vl {

1652 v = strings.ToLower(strings.TrimSpace(v))

1653 if t[1].MatchString(v) {

1654 continue header

1655 }

1656 }

1657 }

1658 continue ruleset

1659 }

1660 return &rs

1661 }

1662 return nil

1663}

1664

1665// MessagePath returns the file system path of a message.

1666func (a *Account) MessagePath(messageID int64) string {

1667 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

1668}

1669

1670// MessageReader opens a message for reading, transparently combining the

1671// message prefix with the original incoming message.

1672func (a *Account) MessageReader(m Message) *MsgReader {

1673 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

1674}

1675

1676// DeliverDestination delivers an email to dest, based on the configured rulesets.

1677//

1678// Caller must hold account wlock (mailbox may be created).

1679// Message delivery, possible mailbox creation, and updated mailbox counts are

1680// broadcasted.

1681func (a *Account) DeliverDestination(log *mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

1682 var mailbox string

1683 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

1684 if rs != nil {

1685 mailbox = rs.Mailbox

1686 } else if dest.Mailbox == "" {

1687 mailbox = "Inbox"

1688 } else {

1689 mailbox = dest.Mailbox

1690 }

1691 return a.DeliverMailbox(log, mailbox, m, msgFile)

1692}

1693

1694// DeliverMailbox delivers an email to the specified mailbox.

1695//

1696// Caller must hold account wlock (mailbox may be created).

1697// Message delivery, possible mailbox creation, and updated mailbox counts are

1698// broadcasted.

1699func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgFile *os.File) error {

1700 var changes []Change

1701 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1702 mb, chl, err := a.MailboxEnsure(tx, mailbox, true)

1703 if err != nil {

1704 return fmt.Errorf("ensuring mailbox: %w", err)

1705 }

1706 m.MailboxID = mb.ID

1707 m.MailboxOrigID = mb.ID

1708

1709 // Update count early, DeliverMessage will update mb too and we don't want to fetch

1710 // it again before updating.

1711 mb.MailboxCounts.Add(m.MailboxCounts())

1712 if err := tx.Update(&mb); err != nil {

1713 return fmt.Errorf("updating mailbox for delivery: %w", err)

1714 }

1715

1716 if err := a.DeliverMessage(log, tx, m, msgFile, true, false, false); err != nil {

1717 return err

1718 }

1719

1720 changes = append(changes, chl...)

1721 changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())

1722 return nil

1723 })

1724 // todo: if rename succeeded but transaction failed, we should remove the file.

1725 if err != nil {

1726 return err

1727 }

1728

1729 BroadcastChanges(a, changes)

1730 return nil

1731}

1732

1733// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

1734//

1735// Caller most hold account wlock.

1736// Changes are broadcasted.

1737func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {

1738 var changes []Change

1739

1740 var remove []Message

1741 defer func() {

1742 for _, m := range remove {

1743 p := a.MessagePath(m.ID)

1744 err := os.Remove(p)

1745 log.Check(err, "removing rejects message file", mlog.Field("path", p))

1746 }

1747 }()

1748

1749 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1750 mb, err := a.MailboxFind(tx, rejectsMailbox)

1751 if err != nil {

1752 return fmt.Errorf("finding mailbox: %w", err)

1753 }

1754 if mb == nil {

1755 // No messages have been delivered yet.

1756 hasSpace = true

1757 return nil

1758 }

1759

1760 // Gather old messages to remove.

1761 old := time.Now().Add(-14 * 24 * time.Hour)

1762 qdel := bstore.QueryTx[Message](tx)

1763 qdel.FilterNonzero(Message{MailboxID: mb.ID})

1764 qdel.FilterEqual("Expunged", false)

1765 qdel.FilterLess("Received", old)

1766 remove, err = qdel.List()

1767 if err != nil {

1768 return fmt.Errorf("listing old messages: %w", err)

1769 }

1770

1771 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

1772 if err != nil {

1773 return fmt.Errorf("removing messages: %w", err)

1774 }

1775

1776 // We allow up to n messages.

1777 qcount := bstore.QueryTx[Message](tx)

1778 qcount.FilterNonzero(Message{MailboxID: mb.ID})

1779 qcount.FilterEqual("Expunged", false)

1780 qcount.Limit(1000)

1781 n, err := qcount.Count()

1782 if err != nil {

1783 return fmt.Errorf("counting rejects: %w", err)

1784 }

1785 hasSpace = n < 1000

1786

1787 return nil

1788 })

1789 if err != nil {

1790 remove = nil // Don't remove files on failure.

1791 return false, err

1792 }

1793

1794 BroadcastChanges(a, changes)

1795

1796 return hasSpace, nil

1797}

1798

1799func (a *Account) rejectsRemoveMessages(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {

1800 if len(l) == 0 {

1801 return nil, nil

1802 }

1803 ids := make([]int64, len(l))

1804 anyids := make([]any, len(l))

1805 for i, m := range l {

1806 ids[i] = m.ID

1807 anyids[i] = m.ID

1808 }

1809

1810 // Remove any message recipients. Should not happen, but a user can move messages

1811 // from a Sent mailbox to the rejects mailbox...

1812 qdmr := bstore.QueryTx[Recipient](tx)

1813 qdmr.FilterEqual("MessageID", anyids...)

1814 if _, err := qdmr.Delete(); err != nil {

1815 return nil, fmt.Errorf("deleting from message recipient: %w", err)

1816 }

1817

1818 // Assign new modseq.

1819 modseq, err := a.NextModSeq(tx)

1820 if err != nil {

1821 return nil, fmt.Errorf("assign next modseq: %w", err)

1822 }

1823

1824 // Expunge the messages.

1825 qx := bstore.QueryTx[Message](tx)

1826 qx.FilterIDs(ids)

1827 var expunged []Message

1828 qx.Gather(&expunged)

1829 if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {

1830 return nil, fmt.Errorf("expunging messages: %w", err)

1831 }

1832

1833 for _, m := range expunged {

1834 m.Expunged = false // Was set by update, but would cause wrong count.

1835 mb.MailboxCounts.Sub(m.MailboxCounts())

1836 }

1837 if err := tx.Update(mb); err != nil {

1838 return nil, fmt.Errorf("updating mailbox counts: %w", err)

1839 }

1840

1841 // Mark as neutral and train so junk filter gets untrained with these (junk) messages.

1842 for i := range expunged {

1843 expunged[i].Junk = false

1844 expunged[i].Notjunk = false

1845 }

1846 if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {

1847 return nil, fmt.Errorf("retraining expunged messages: %w", err)

1848 }

1849

1850 changes := make([]Change, len(l), len(l)+1)

1851 for i, m := range l {

1852 changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}

1853 }

1854 changes = append(changes, mb.ChangeCounts())

1855 return changes, nil

1856}

1857

1858// RejectsRemove removes a message from the rejects mailbox if present.

1859// Caller most hold account wlock.

1860// Changes are broadcasted.

1861func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string) error {

1862 var changes []Change

1863

1864 var remove []Message

1865 defer func() {

1866 for _, m := range remove {

1867 p := a.MessagePath(m.ID)

1868 err := os.Remove(p)

1869 log.Check(err, "removing rejects message file", mlog.Field("path", p))

1870 }

1871 }()

1872

1873 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1874 mb, err := a.MailboxFind(tx, rejectsMailbox)

1875 if err != nil {

1876 return fmt.Errorf("finding mailbox: %w", err)

1877 }

1878 if mb == nil {

1879 return nil

1880 }

1881

1882 q := bstore.QueryTx[Message](tx)

1883 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

1884 q.FilterEqual("Expunged", false)

1885 remove, err = q.List()

1886 if err != nil {

1887 return fmt.Errorf("listing messages to remove: %w", err)

1888 }

1889

1890 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

1891 if err != nil {

1892 return fmt.Errorf("removing messages: %w", err)

1893 }

1894

1895 return nil

1896 })

1897 if err != nil {

1898 remove = nil // Don't remove files on failure.

1899 return err

1900 }

1901

1902 BroadcastChanges(a, changes)

1903

1904 return nil

1905}

1906

1907// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

1908var authCache = struct {

1909 sync.Mutex

1910 success map[authKey]string

1911}{

1912 success: map[authKey]string{},

1913}

1914

1915type authKey struct {

1916 email, hash string

1917}

1918

1919// StartAuthCache starts a goroutine that regularly clears the auth cache.

1920func StartAuthCache() {

1921 go manageAuthCache()

1922}

1923

1924func manageAuthCache() {

1925 for {

1926 authCache.Lock()

1927 authCache.success = map[authKey]string{}

1928 authCache.Unlock()

1929 time.Sleep(15 * time.Minute)

1930 }

1931}

1932

1933// OpenEmailAuth opens an account given an email address and password.

1934//

1935// The email address may contain a catchall separator.

1936func OpenEmailAuth(email string, password string) (acc *Account, rerr error) {

1937 acc, _, rerr = OpenEmail(email)

1938 if rerr != nil {

1939 return

1940 }

1941

1942 defer func() {

1943 if rerr != nil && acc != nil {

1944 err := acc.Close()

1945 xlog.Check(err, "closing account after open auth failure")

1946 acc = nil

1947 }

1948 }()

1949

1950 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

1951 if err != nil {

1952 if err == bstore.ErrAbsent {

1953 return acc, ErrUnknownCredentials

1954 }

1955 return acc, fmt.Errorf("looking up password: %v", err)

1956 }

1957 authCache.Lock()

1958 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

1959 authCache.Unlock()

1960 if ok {

1961 return

1962 }

1963 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

1964 rerr = ErrUnknownCredentials

1965 } else {

1966 authCache.Lock()

1967 authCache.success[authKey{email, pw.Hash}] = password

1968 authCache.Unlock()

1969 }

1970 return

1971}

1972

1973// OpenEmail opens an account given an email address.

1974//

1975// The email address may contain a catchall separator.

1976func OpenEmail(email string) (*Account, config.Destination, error) {

1977 addr, err := smtp.ParseAddress(email)

1978 if err != nil {

1979 return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

1980 }

1981 accountName, _, dest, err := mox.FindAccount(addr.Localpart, addr.Domain, false)

1982 if err != nil && (errors.Is(err, mox.ErrAccountNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

1983 return nil, config.Destination{}, ErrUnknownCredentials

1984 } else if err != nil {

1985 return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)

1986 }

1987 acc, err := OpenAccount(accountName)

1988 if err != nil {

1989 return nil, config.Destination{}, err

1990 }

1991 return acc, dest, nil

1992}

1993

1994// 64 characters, must be power of 2 for MessagePath

1995const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

1996

1997// MessagePath returns the filename of the on-disk filename, relative to the

1998// containing directory such as <account>/msg or queue.

1999// Returns names like "AB/1".

2000func MessagePath(messageID int64) string {

2001 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

2002}

2003

2004// messagePathElems returns the elems, for a single join without intermediate

2005// string allocations.

2006func messagePathElems(messageID int64) []string {

2007 v := messageID >> 13 // 8k files per directory.

2008 dir := ""

2009 for {

2010 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

2011 v >>= 6

2012 if v == 0 {

2013 break

2014 }

2015 }

2016 return []string{dir, strconv.FormatInt(messageID, 10)}

2017}

2018

2019// Set returns a copy of f, with each flag that is true in mask set to the

2020// value from flags.

2021func (f Flags) Set(mask, flags Flags) Flags {

2022 set := func(d *bool, m, v bool) {

2023 if m {

2024 *d = v

2025 }

2026 }

2027 r := f

2028 set(&r.Seen, mask.Seen, flags.Seen)

2029 set(&r.Answered, mask.Answered, flags.Answered)

2030 set(&r.Flagged, mask.Flagged, flags.Flagged)

2031 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

2032 set(&r.Junk, mask.Junk, flags.Junk)

2033 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

2034 set(&r.Deleted, mask.Deleted, flags.Deleted)

2035 set(&r.Draft, mask.Draft, flags.Draft)

2036 set(&r.Phishing, mask.Phishing, flags.Phishing)

2037 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

2038 return r

2039}

2040

2041// Changed returns a mask of flags that have been between f and other.

2042func (f Flags) Changed(other Flags) (mask Flags) {

2043 mask.Seen = f.Seen != other.Seen

2044 mask.Answered = f.Answered != other.Answered

2045 mask.Flagged = f.Flagged != other.Flagged

2046 mask.Forwarded = f.Forwarded != other.Forwarded

2047 mask.Junk = f.Junk != other.Junk

2048 mask.Notjunk = f.Notjunk != other.Notjunk

2049 mask.Deleted = f.Deleted != other.Deleted

2050 mask.Draft = f.Draft != other.Draft

2051 mask.Phishing = f.Phishing != other.Phishing

2052 mask.MDNSent = f.MDNSent != other.MDNSent

2053 return

2054}

2055

2056var systemWellKnownFlags = map[string]bool{

2057 `\answered`: true,

2058 `\flagged`: true,

2059 `\deleted`: true,

2060 `\seen`: true,

2061 `\draft`: true,

2062 `$junk`: true,

2063 `$notjunk`: true,

2064 `$forwarded`: true,

2065 `$phishing`: true,

2066 `$mdnsent`: true,

2067}

2068

2069// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

2070// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

2071func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

2072 fields := map[string]*bool{

2073 `\answered`: &flags.Answered,

2074 `\flagged`: &flags.Flagged,

2075 `\deleted`: &flags.Deleted,

2076 `\seen`: &flags.Seen,

2077 `\draft`: &flags.Draft,

2078 `$junk`: &flags.Junk,

2079 `$notjunk`: &flags.Notjunk,

2080 `$forwarded`: &flags.Forwarded,

2081 `$phishing`: &flags.Phishing,

2082 `$mdnsent`: &flags.MDNSent,

2083 }

2084 seen := map[string]bool{}

2085 for _, f := range l {

2086 f = strings.ToLower(f)

2087 if field, ok := fields[f]; ok {

2088 *field = true

2089 } else if seen[f] {

2090 if moxvar.Pedantic {

2091 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

2092 }

2093 } else {

2094 if err := CheckKeyword(f); err != nil {

2095 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

2096 }

2097 keywords = append(keywords, f)

2098 seen[f] = true

2099 }

2100 }

2101 sort.Strings(keywords)

2102 return flags, keywords, nil

2103}

2104

2105// RemoveKeywords removes keywords from l, returning whether any modifications were

2106// made, and a slice, a new slice in case of modifications. Keywords must have been

2107// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

2108// be used with valid keywords, not with system flags like \Seen.

2109func RemoveKeywords(l, remove []string) ([]string, bool) {

2110 var copied bool

2111 var changed bool

2112 for _, k := range remove {

2113 if i := slices.Index(l, k); i >= 0 {

2114 if !copied {

2115 l = append([]string{}, l...)

2116 copied = true

2117 }

2118 copy(l[i:], l[i+1:])

2119 l = l[:len(l)-1]

2120 changed = true

2121 }

2122 }

2123 return l, changed

2124}

2125

2126// MergeKeywords adds keywords from add into l, returning whether it added any

2127// keyword, and the slice with keywords, a new slice if modifications were made.

2128// Keywords are only added if they aren't already present. Should only be used with

2129// keywords, not with system flags like \Seen.

2130func MergeKeywords(l, add []string) ([]string, bool) {

2131 var copied bool

2132 var changed bool

2133 for _, k := range add {

2134 if !slices.Contains(l, k) {

2135 if !copied {

2136 l = append([]string{}, l...)

2137 copied = true

2138 }

2139 l = append(l, k)

2140 changed = true

2141 }

2142 }

2143 if changed {

2144 sort.Strings(l)

2145 }

2146 return l, changed

2147}

2148

2149// CheckKeyword returns an error if kw is not a valid keyword. Kw should

2150// already be in lower-case.

2151func CheckKeyword(kw string) error {

2152 if kw == "" {

2153 return fmt.Errorf("keyword cannot be empty")

2154 }

2155 if systemWellKnownFlags[kw] {

2156 return fmt.Errorf("cannot use well-known flag as keyword")

2157 }

2158 for _, c := range kw {

2159 // ../rfc/9051:6334

2160 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

2161 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

2162 }

2163 }

2164 return nil

2165}

2166

2167// SendLimitReached checks whether sending a message to recipients would reach

2168// the limit of outgoing messages for the account. If so, the message should

2169// not be sent. If the returned numbers are >= 0, the limit was reached and the

2170// values are the configured limits.

2171//

2172// To limit damage to the internet and our reputation in case of account

2173// compromise, we limit the max number of messages sent in a 24 hour window, both

2174// total number of messages and number of first-time recipients.

2175func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

2176 conf, _ := a.Conf()

2177 msgmax := conf.MaxOutgoingMessagesPerDay

2178 if msgmax == 0 {

2179 // For human senders, 1000 recipients in a day is quite a lot.

2180 msgmax = 1000

2181 }

2182 rcptmax := conf.MaxFirstTimeRecipientsPerDay

2183 if rcptmax == 0 {

2184 // Human senders may address a new human-sized list of people once in a while. In

2185 // case of a compromise, a spammer will probably try to send to many new addresses.

2186 rcptmax = 200

2187 }

2188

2189 rcpts := map[string]time.Time{}

2190 n := 0

2191 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

2192 n++

2193 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

2194 rcpts[o.Recipient] = o.Submitted

2195 }

2196 return nil

2197 })

2198 if err != nil {

2199 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

2200 }

2201 if n+len(recipients) > msgmax {

2202 return msgmax, -1, nil

2203 }

2204

2205 // Only check if max first-time recipients is reached if there are enough messages

2206 // to trigger the limit.

2207 if n+len(recipients) < rcptmax {

2208 return -1, -1, nil

2209 }

2210

2211 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

2212 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

2213 return !exists, err

2214 }

2215

2216 firsttime := 0

2217 now := time.Now()

2218 for _, r := range recipients {

2219 if first, err := isFirstTime(r.XString(true), now); err != nil {

2220 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2221 } else if first {

2222 firsttime++

2223 }

2224 }

2225 for r, t := range rcpts {

2226 if first, err := isFirstTime(r, t); err != nil {

2227 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2228 } else if first {

2229 firsttime++

2230 }

2231 }

2232 if firsttime > rcptmax {

2233 return -1, rcptmax, nil

2234 }

2235 return -1, -1, nil

2236}

2237

2238// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

2239// the total list of created mailboxes is returned in created. On success, if

2240// exists is false and rerr nil, the changes must be broadcasted by the caller.

2241//

2242// Name must be in normalized form.

2243func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, created []string, exists bool, rerr error) {

2244 elems := strings.Split(name, "/")

2245 var p string

2246 for i, elem := range elems {

2247 if i > 0 {

2248 p += "/"

2249 }

2250 p += elem

2251 exists, err := a.MailboxExists(tx, p)

2252 if err != nil {

2253 return nil, nil, false, fmt.Errorf("checking if mailbox exists")

2254 }

2255 if exists {

2256 if i == len(elems)-1 {

2257 return nil, nil, true, fmt.Errorf("mailbox already exists")

2258 }

2259 continue

2260 }

2261 _, nchanges, err := a.MailboxEnsure(tx, p, true)

2262 if err != nil {

2263 return nil, nil, false, fmt.Errorf("ensuring mailbox exists")

2264 }

2265 changes = append(changes, nchanges...)

2266 created = append(created, p)

2267 }

2268 return changes, created, false, nil

2269}

2270

2271// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the

2272// destination, and any children of mbsrc and the destination.

2273//

2274// Names must be normalized and cannot be Inbox.

2275func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) {

2276 if mbsrc.Name == "Inbox" || dst == "Inbox" {

2277 return nil, true, false, false, fmt.Errorf("inbox cannot be renamed")

2278 }

2279

2280 // We gather existing mailboxes that we need for deciding what to create/delete/update.

2281 q := bstore.QueryTx[Mailbox](tx)

2282 srcPrefix := mbsrc.Name + "/"

2283 dstRoot := strings.SplitN(dst, "/", 2)[0]

2284 dstRootPrefix := dstRoot + "/"

2285 q.FilterFn(func(mb Mailbox) bool {

2286 return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix)

2287 })

2288 q.SortAsc("Name") // We'll rename the parents before children.

2289 l, err := q.List()

2290 if err != nil {

2291 return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err)

2292 }

2293

2294 mailboxes := map[string]Mailbox{}

2295 for _, mb := range l {

2296 mailboxes[mb.Name] = mb

2297 }

2298

2299 if _, ok := mailboxes[mbsrc.Name]; !ok {

2300 return nil, false, true, false, fmt.Errorf("mailbox does not exist")

2301 }

2302

2303 uidval, err := a.NextUIDValidity(tx)

2304 if err != nil {

2305 return nil, false, false, false, fmt.Errorf("next uid validity: %v", err)

2306 }

2307

2308 // Ensure parent mailboxes for the destination paths exist.

2309 var parent string

2310 dstElems := strings.Split(dst, "/")

2311 for i, elem := range dstElems[:len(dstElems)-1] {

2312 if i > 0 {

2313 parent += "/"

2314 }

2315 parent += elem

2316

2317 mb, ok := mailboxes[parent]

2318 if ok {

2319 continue

2320 }

2321 omb := mb

2322 mb = Mailbox{

2323 ID: omb.ID,

2324 Name: parent,

2325 UIDValidity: uidval,

2326 UIDNext: 1,

2327 HaveCounts: true,

2328 }

2329 if err := tx.Insert(&mb); err != nil {

2330 return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err)

2331 }

2332 if err := tx.Get(&Subscription{Name: parent}); err != nil {

2333 if err := tx.Insert(&Subscription{Name: parent}); err != nil {

2334 return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err)

2335 }

2336 }

2337 changes = append(changes, ChangeAddMailbox{Mailbox: mb, Flags: []string{`\Subscribed`}})

2338 }

2339

2340 // Process src mailboxes, renaming them to dst.

2341 for _, srcmb := range l {

2342 if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) {

2343 continue

2344 }

2345 srcName := srcmb.Name

2346 dstName := dst + srcmb.Name[len(mbsrc.Name):]

2347 if _, ok := mailboxes[dstName]; ok {

2348 return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName)

2349 }

2350

2351 srcmb.Name = dstName

2352 srcmb.UIDValidity = uidval

2353 if err := tx.Update(&srcmb); err != nil {

2354 return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err)

2355 }

2356

2357 var dstFlags []string

2358 if tx.Get(&Subscription{Name: dstName}) == nil {

2359 dstFlags = []string{`\Subscribed`}

2360 }

2361 changes = append(changes, ChangeRenameMailbox{MailboxID: srcmb.ID, OldName: srcName, NewName: dstName, Flags: dstFlags})

2362 }

2363

2364 // If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c.

2365 srcElems := strings.Split(mbsrc.Name, "/")

2366 xsrc := mbsrc.Name

2367 for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ {

2368 mb := Mailbox{

2369 UIDValidity: uidval,

2370 UIDNext: 1,

2371 Name: xsrc,

2372 HaveCounts: true,

2373 }

2374 if err := tx.Insert(&mb); err != nil {

2375 return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err)

2376 }

2377 xsrc += "/" + dstElems[len(srcElems)+i]

2378 }

2379 return changes, false, false, false, nil

2380}

2381

2382// MailboxDelete deletes a mailbox by ID. If it has children, the return value

2383// indicates that and an error is returned.

2384//

2385// Caller should broadcast the changes and remove files for the removed message IDs.

2386func (a *Account) MailboxDelete(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {

2387 // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about

2388 // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.

2389 qmb := bstore.QueryTx[Mailbox](tx)

2390 mbprefix := mailbox.Name + "/"

2391 qmb.FilterFn(func(mb Mailbox) bool {

2392 return strings.HasPrefix(mb.Name, mbprefix)

2393 })

2394 if childExists, err := qmb.Exists(); err != nil {

2395 return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err)

2396 } else if childExists {

2397 return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")

2398 }

2399

2400 // todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged.

2401

2402 qm := bstore.QueryTx[Message](tx)

2403 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2404 remove, err := qm.List()

2405 if err != nil {

2406 return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err)

2407 }

2408

2409 if len(remove) > 0 {

2410 removeIDs := make([]any, len(remove))

2411 for i, m := range remove {

2412 removeIDs[i] = m.ID

2413 }

2414 qmr := bstore.QueryTx[Recipient](tx)

2415 qmr.FilterEqual("MessageID", removeIDs...)

2416 if _, err = qmr.Delete(); err != nil {

2417 return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err)

2418 }

2419

2420 qm = bstore.QueryTx[Message](tx)

2421 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2422 if _, err := qm.Delete(); err != nil {

2423 return nil, nil, false, fmt.Errorf("removing messages: %v", err)

2424 }

2425

2426 for _, m := range remove {

2427 if !m.Expunged {

2428 removeMessageIDs = append(removeMessageIDs, m.ID)

2429 }

2430 }

2431

2432 // Mark messages as not needing training. Then retrain them, so they are untrained if they were.

2433 n := 0

2434 o := 0

2435 for _, m := range remove {

2436 if !m.Expunged {

2437 remove[o] = m

2438 remove[o].Junk = false

2439 remove[o].Notjunk = false

2440 n++

2441 }

2442 }

2443 remove = remove[:n]

2444 if err := a.RetrainMessages(ctx, log, tx, remove, true); err != nil {

2445 return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err)

2446 }

2447 }

2448

2449 if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {

2450 return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)

2451 }

2452 return []Change{ChangeRemoveMailbox{MailboxID: mailbox.ID, Name: mailbox.Name}}, removeMessageIDs, false, nil

2453}

2454

2455// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.

2456// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.

2457// Name is invalid if it contains leading/trailing/double slashes, or when it isn't

2458// unicode-normalized, or when empty or has special characters.

2459//

2460// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.

2461// For that case, and for other invalid names, an error is returned.

2462func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {

2463 first := strings.SplitN(name, "/", 2)[0]

2464 if strings.EqualFold(first, "inbox") {

2465 if len(name) == len("inbox") && !allowInbox {

2466 return "", true, fmt.Errorf("special mailbox name Inbox not allowed")

2467 }

2468 name = "Inbox" + name[len("Inbox"):]

2469 }

2470

2471 if norm.NFC.String(name) != name {

2472 return "", false, errors.New("non-unicode-normalized mailbox names not allowed")

2473 }

2474

2475 if name == "" {

2476 return "", false, errors.New("empty mailbox name")

2477 }

2478 if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {

2479 return "", false, errors.New("bad slashes in mailbox name")

2480 }

2481 for _, c := range name {

2482 switch c {

2483 case '%', '*', '#', '&':

2484 return "", false, fmt.Errorf("character %c not allowed in mailbox name", c)

2485 }

2486 // ../rfc/6855:192

2487 if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {

2488 return "", false, errors.New("control characters not allowed in mailbox name")

2489 }

2490 }

2491 return name, false, nil

2492}

2493