1package webmail

2

3import (

4 "context"

5 "encoding/base64"

6 "encoding/json"

7 "errors"

8 "fmt"

9 "io"

10 "mime"

11 "mime/multipart"

12 "net"

13 "net/http"

14 "net/mail"

15 "net/textproto"

16 "os"

17 "runtime/debug"

18 "sort"

19 "strings"

20 "sync"

21 "time"

22

23 _ "embed"

24

25 "golang.org/x/exp/maps"

26

27 "github.com/mjl-/bstore"

28 "github.com/mjl-/sherpa"

29 "github.com/mjl-/sherpadoc"

30 "github.com/mjl-/sherpaprom"

31

32 "github.com/mjl-/mox/dkim"

33 "github.com/mjl-/mox/dns"

34 "github.com/mjl-/mox/message"

35 "github.com/mjl-/mox/metrics"

36 "github.com/mjl-/mox/mlog"

37 "github.com/mjl-/mox/mox-"

38 "github.com/mjl-/mox/moxio"

39 "github.com/mjl-/mox/moxvar"

40 "github.com/mjl-/mox/mtasts"

41 "github.com/mjl-/mox/mtastsdb"

42 "github.com/mjl-/mox/queue"

43 "github.com/mjl-/mox/smtp"

44 "github.com/mjl-/mox/smtpclient"

45 "github.com/mjl-/mox/store"

46)

47

48//go:embed api.json

49var webmailapiJSON []byte

50

51type Webmail struct {

52 maxMessageSize int64 // From listener.

53}

54

55func mustParseAPI(api string, buf []byte) (doc sherpadoc.Section) {

56 err := json.Unmarshal(buf, &doc)

57 if err != nil {

58 xlog.Fatalx("parsing webmail api docs", err, mlog.Field("api", api))

59 }

60 return doc

61}

62

63var webmailDoc = mustParseAPI("webmail", webmailapiJSON)

64

65var sherpaHandlerOpts *sherpa.HandlerOpts

66

67func makeSherpaHandler(maxMessageSize int64) (http.Handler, error) {

68 return sherpa.NewHandler("/api/", moxvar.Version, Webmail{maxMessageSize}, &webmailDoc, sherpaHandlerOpts)

69}

70

71func init() {

72 collector, err := sherpaprom.NewCollector("moxwebmail", nil)

73 if err != nil {

74 xlog.Fatalx("creating sherpa prometheus collector", err)

75 }

76

77 sherpaHandlerOpts = &sherpa.HandlerOpts{Collector: collector, AdjustFunctionNames: "none"}

78 // Just to validate.

79 _, err = makeSherpaHandler(0)

80 if err != nil {

81 xlog.Fatalx("sherpa handler", err)

82 }

83}

84

85// Token returns a token to use for an SSE connection. A token can only be used for

86// a single SSE connection. Tokens are stored in memory for a maximum of 1 minute,

87// with at most 10 unused tokens (the most recently created) per account.

88func (Webmail) Token(ctx context.Context) string {

89 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

90 return sseTokens.xgenerate(ctx, reqInfo.AccountName, reqInfo.LoginAddress)

91}

92

93// Requests sends a new request for an open SSE connection. Any currently active

94// request for the connection will be canceled, but this is done asynchrously, so

95// the SSE connection may still send results for the previous request. Callers

96// should take care to ignore such results. If req.Cancel is set, no new request is

97// started.

98func (Webmail) Request(ctx context.Context, req Request) {

99 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

100

101 if !req.Cancel && req.Page.Count <= 0 {

102 xcheckuserf(ctx, errors.New("Page.Count must be >= 1"), "checking request")

103 }

104

105 sse, ok := sseGet(req.SSEID, reqInfo.AccountName)

106 if !ok {

107 xcheckuserf(ctx, errors.New("unknown sseid"), "looking up connection")

108 }

109 sse.Request <- req

110}

111

112// ParsedMessage returns enough to render the textual body of a message. It is

113// assumed the client already has other fields through MessageItem.

114func (Webmail) ParsedMessage(ctx context.Context, msgID int64) (pm ParsedMessage) {

115 log := xlog.WithContext(ctx)

116 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

117 acc, err := store.OpenAccount(reqInfo.AccountName)

118 xcheckf(ctx, err, "open account")

119 defer func() {

120 err := acc.Close()

121 log.Check(err, "closing account")

122 }()

123

124 var m store.Message

125 xdbread(ctx, acc, func(tx *bstore.Tx) {

126 m = xmessageID(ctx, tx, msgID)

127 })

128

129 state := msgState{acc: acc}

130 defer state.clear()

131 pm, err = parsedMessage(log, m, &state, true, false)

132 xcheckf(ctx, err, "parsing message")

133 return

134}

135

136// Attachment is a MIME part is an existing message that is not intended as

137// viewable text or HTML part.

138type Attachment struct {

139 Path []int // Indices into top-level message.Part.Parts.

140

141 // File name based on "name" attribute of "Content-Type", or the "filename"

142 // attribute of "Content-Disposition".

143 Filename string

144

145 Part message.Part

146}

147

148// SubmitMessage is an email message to be sent to one or more recipients.

149// Addresses are formatted as just email address, or with a name like "name

150// <user@host>".

151type SubmitMessage struct {

152 From string

153 To []string

154 Cc []string

155 Bcc []string

156 Subject string

157 TextBody string

158 Attachments []File

159 ForwardAttachments ForwardAttachments

160 IsForward bool

161 ResponseMessageID int64 // If set, this was a reply or forward, based on IsForward.

162 ReplyTo string // If non-empty, Reply-To header to add to message.

163 UserAgent string // User-Agent header added if not empty.

164 RequireTLS *bool // For "Require TLS" extension during delivery.

165}

166

167// ForwardAttachments references attachments by a list of message.Part paths.

168type ForwardAttachments struct {

169 MessageID int64 // Only relevant if MessageID is not 0.

170 Paths [][]int // List of attachments, each path is a list of indices into the top-level message.Part.Parts.

171}

172

173// File is a new attachment (not from an existing message that is being

174// forwarded) to send with a SubmitMessage.

175type File struct {

176 Filename string

177 DataURI string // Full data of the attachment, with base64 encoding and including content-type.

178}

179

180// parseAddress expects either a plain email address like "user@domain", or a

181// single address as used in a message header, like "name <user@domain>".

182func parseAddress(msghdr string) (message.NameAddress, error) {

183 a, err := mail.ParseAddress(msghdr)

184 if err != nil {

185 return message.NameAddress{}, nil

186 }

187

188 // todo: parse more fully according to ../rfc/5322:959

189 path, err := smtp.ParseAddress(a.Address)

190 if err != nil {

191 return message.NameAddress{}, err

192 }

193 return message.NameAddress{DisplayName: a.Name, Address: path}, nil

194}

195

196func xmailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) store.Mailbox {

197 if mailboxID == 0 {

198 xcheckuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox")

199 }

200 mb := store.Mailbox{ID: mailboxID}

201 err := tx.Get(&mb)

202 if err == bstore.ErrAbsent {

203 xcheckuserf(ctx, err, "getting mailbox")

204 }

205 xcheckf(ctx, err, "getting mailbox")

206 return mb

207}

208

209// xmessageID returns a non-expunged message or panics with a sherpa error.

210func xmessageID(ctx context.Context, tx *bstore.Tx, messageID int64) store.Message {

211 if messageID == 0 {

212 xcheckuserf(ctx, errors.New("invalid zero message id"), "getting message")

213 }

214 m := store.Message{ID: messageID}

215 err := tx.Get(&m)

216 if err == bstore.ErrAbsent {

217 xcheckuserf(ctx, errors.New("message does not exist"), "getting message")

218 } else if err == nil && m.Expunged {

219 xcheckuserf(ctx, errors.New("message was removed"), "getting message")

220 }

221 xcheckf(ctx, err, "getting message")

222 return m

223}

224

225// MessageSubmit sends a message by submitting it the outgoing email queue. The

226// message is sent to all addresses listed in the To, Cc and Bcc addresses, without

227// Bcc message header.

228//

229// If a Sent mailbox is configured, messages are added to it after submitting

230// to the delivery queue.

231func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {

232 // Similar between ../smtpserver/server.go:/submit\( and ../webmail/webmail.go:/MessageSubmit\(

233

234 // todo: consider making this an HTTP POST, so we can upload as regular form, which is probably more efficient for encoding for the client and we can stream the data in.

235

236 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

237 log := xlog.WithContext(ctx).Fields(mlog.Field("account", reqInfo.AccountName))

238 acc, err := store.OpenAccount(reqInfo.AccountName)

239 xcheckf(ctx, err, "open account")

240 defer func() {

241 err := acc.Close()

242 log.Check(err, "closing account")

243 }()

244

245 log.Debug("message submit")

246

247 fromAddr, err := parseAddress(m.From)

248 xcheckuserf(ctx, err, "parsing From address")

249

250 var replyTo *message.NameAddress

251 if m.ReplyTo != "" {

252 a, err := parseAddress(m.ReplyTo)

253 xcheckuserf(ctx, err, "parsing Reply-To address")

254 replyTo = &a

255 }

256

257 var recipients []smtp.Address

258

259 var toAddrs []message.NameAddress

260 for _, s := range m.To {

261 addr, err := parseAddress(s)

262 xcheckuserf(ctx, err, "parsing To address")

263 toAddrs = append(toAddrs, addr)

264 recipients = append(recipients, addr.Address)

265 }

266

267 var ccAddrs []message.NameAddress

268 for _, s := range m.Cc {

269 addr, err := parseAddress(s)

270 xcheckuserf(ctx, err, "parsing Cc address")

271 ccAddrs = append(ccAddrs, addr)

272 recipients = append(recipients, addr.Address)

273 }

274

275 for _, s := range m.Bcc {

276 addr, err := parseAddress(s)

277 xcheckuserf(ctx, err, "parsing Bcc address")

278 recipients = append(recipients, addr.Address)

279 }

280

281 // Check if from address is allowed for account.

282 fromAccName, _, _, err := mox.FindAccount(fromAddr.Address.Localpart, fromAddr.Address.Domain, false)

283 if err == nil && fromAccName != reqInfo.AccountName {

284 err = mox.ErrAccountNotFound

285 }

286 if err != nil && (errors.Is(err, mox.ErrAccountNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

287 metricSubmission.WithLabelValues("badfrom").Inc()

288 xcheckuserf(ctx, errors.New("address not found"), "looking from address for account")

289 }

290 xcheckf(ctx, err, "checking if from address is allowed")

291

292 if len(recipients) == 0 {

293 xcheckuserf(ctx, fmt.Errorf("no recipients"), "composing message")

294 }

295

296 // Check outgoing message rate limit.

297 xdbread(ctx, acc, func(tx *bstore.Tx) {

298 rcpts := make([]smtp.Path, len(recipients))

299 for i, r := range recipients {

300 rcpts[i] = smtp.Path{Localpart: r.Localpart, IPDomain: dns.IPDomain{Domain: r.Domain}}

301 }

302 msglimit, rcptlimit, err := acc.SendLimitReached(tx, rcpts)

303 if msglimit >= 0 {

304 metricSubmission.WithLabelValues("messagelimiterror").Inc()

305 xcheckuserf(ctx, errors.New("send message limit reached"), "checking outgoing rate limit")

306 } else if rcptlimit >= 0 {

307 metricSubmission.WithLabelValues("recipientlimiterror").Inc()

308 xcheckuserf(ctx, errors.New("send message limit reached"), "checking outgoing rate limit")

309 }

310 xcheckf(ctx, err, "checking send limit")

311 })

312

313 has8bit := false // We update this later on.

314

315 // We only use smtputf8 if we have to, with a utf-8 localpart. For IDNA, we use ASCII domains.

316 smtputf8 := false

317 for _, a := range recipients {

318 if a.Localpart.IsInternational() {

319 smtputf8 = true

320 break

321 }

322 }

323 if !smtputf8 && fromAddr.Address.Localpart.IsInternational() {

324 // todo: may want to warn user that they should consider sending with a ascii-only localpart, in case receiver doesn't support smtputf8.

325 smtputf8 = true

326 }

327

328 // Create file to compose message into.

329 dataFile, err := store.CreateMessageTemp("webmail-submit")

330 xcheckf(ctx, err, "creating temporary file for message")

331 defer store.CloseRemoveTempFile(log, dataFile, "message to submit")

332

333 // If writing to the message file fails, we abort immediately.

334 xc := message.NewComposer(dataFile, w.maxMessageSize)

335 defer func() {

336 x := recover()

337 if x == nil {

338 return

339 }

340 if err, ok := x.(error); ok && errors.Is(err, message.ErrMessageSize) {

341 xcheckuserf(ctx, err, "making message")

342 } else if ok && errors.Is(err, message.ErrCompose) {

343 xcheckf(ctx, err, "making message")

344 }

345 panic(x)

346 }()

347

348 // todo spec: can we add an Authentication-Results header that indicates this is an authenticated message? the "auth" method is for SMTP AUTH, which this isn't. ../rfc/8601 https://www.iana.org/assignments/email-auth/email-auth.xhtml

349

350 // Each queued message gets a Received header.

351 // We don't have access to the local IP for adding.

352 // We cannot use VIA, because there is no registered method. We would like to use

353 // it to add the ascii domain name in case of smtputf8 and IDNA host name.

354 recvFrom := message.HeaderCommentDomain(mox.Conf.Static.HostnameDomain, smtputf8)

355 recvBy := mox.Conf.Static.HostnameDomain.XName(smtputf8)

356 recvID := mox.ReceivedID(mox.CidFromCtx(ctx))

357 recvHdrFor := func(rcptTo string) string {

358 recvHdr := &message.HeaderWriter{}

359 // For additional Received-header clauses, see:

360 // https://www.iana.org/assignments/mail-parameters/mail-parameters.xhtml#table-mail-parameters-8

361 // Note: we don't have "via" or "with", there is no registered for webmail.

362 recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "id", recvID) // ../rfc/5321:3158

363 if reqInfo.Request.TLS != nil {

364 recvHdr.Add(" ", message.TLSReceivedComment(log, *reqInfo.Request.TLS)...)

365 }

366 recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z))

367 return recvHdr.String()

368 }

369

370 // Outer message headers.

371 xc.HeaderAddrs("From", []message.NameAddress{fromAddr})

372 if replyTo != nil {

373 xc.HeaderAddrs("Reply-To", []message.NameAddress{*replyTo})

374 }

375 xc.HeaderAddrs("To", toAddrs)

376 xc.HeaderAddrs("Cc", ccAddrs)

377 if m.Subject != "" {

378 xc.Subject(m.Subject)

379 }

380

381 messageID := fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8))

382 xc.Header("Message-Id", messageID)

383 xc.Header("Date", time.Now().Format(message.RFC5322Z))

384 // Add In-Reply-To and References headers.

385 if m.ResponseMessageID > 0 {

386 xdbread(ctx, acc, func(tx *bstore.Tx) {

387 rm := xmessageID(ctx, tx, m.ResponseMessageID)

388 msgr := acc.MessageReader(rm)

389 defer func() {

390 err := msgr.Close()

391 log.Check(err, "closing message reader")

392 }()

393 rp, err := rm.LoadPart(msgr)

394 xcheckf(ctx, err, "load parsed message")

395 h, err := rp.Header()

396 xcheckf(ctx, err, "parsing header")

397

398 if rp.Envelope == nil {

399 return

400 }

401 xc.Header("In-Reply-To", rp.Envelope.MessageID)

402 ref := h.Get("References")

403 if ref == "" {

404 ref = h.Get("In-Reply-To")

405 }

406 if ref != "" {

407 xc.Header("References", ref+"\r\n\t"+rp.Envelope.MessageID)

408 } else {

409 xc.Header("References", rp.Envelope.MessageID)

410 }

411 })

412 }

413 if m.UserAgent != "" {

414 xc.Header("User-Agent", m.UserAgent)

415 }

416 if m.RequireTLS != nil && !*m.RequireTLS {

417 xc.Header("TLS-Required", "No")

418 }

419 xc.Header("MIME-Version", "1.0")

420

421 if len(m.Attachments) > 0 || len(m.ForwardAttachments.Paths) > 0 {

422 mp := multipart.NewWriter(xc)

423 xc.Header("Content-Type", fmt.Sprintf(`multipart/mixed; boundary="%s"`, mp.Boundary()))

424 xc.Line()

425

426 textBody, ct, cte := xc.TextPart(m.TextBody)

427 textHdr := textproto.MIMEHeader{}

428 textHdr.Set("Content-Type", ct)

429 textHdr.Set("Content-Transfer-Encoding", cte)

430

431 textp, err := mp.CreatePart(textHdr)

432 xcheckf(ctx, err, "adding text part to message")

433 _, err = textp.Write(textBody)

434 xcheckf(ctx, err, "writing text part")

435

436 xaddPart := func(ct, filename string) io.Writer {

437 ahdr := textproto.MIMEHeader{}

438 cd := mime.FormatMediaType("attachment", map[string]string{"filename": filename})

439

440 ahdr.Set("Content-Type", ct)

441 ahdr.Set("Content-Transfer-Encoding", "base64")

442 ahdr.Set("Content-Disposition", cd)

443 ap, err := mp.CreatePart(ahdr)

444 xcheckf(ctx, err, "adding attachment part to message")

445 return ap

446 }

447

448 xaddAttachmentBase64 := func(ct, filename string, base64Data []byte) {

449 ap := xaddPart(ct, filename)

450

451 for len(base64Data) > 0 {

452 line := base64Data

453 n := len(line)

454 if n > 78 {

455 n = 78

456 }

457 line, base64Data = base64Data[:n], base64Data[n:]

458 _, err := ap.Write(line)

459 xcheckf(ctx, err, "writing attachment")

460 _, err = ap.Write([]byte("\r\n"))

461 xcheckf(ctx, err, "writing attachment")

462 }

463 }

464

465 xaddAttachment := func(ct, filename string, r io.Reader) {

466 ap := xaddPart(ct, filename)

467 wc := moxio.Base64Writer(ap)

468 _, err := io.Copy(wc, r)

469 xcheckf(ctx, err, "adding attachment")

470 err = wc.Close()

471 xcheckf(ctx, err, "flushing attachment")

472 }

473

474 for _, a := range m.Attachments {

475 s := a.DataURI

476 if !strings.HasPrefix(s, "data:") {

477 xcheckuserf(ctx, errors.New("missing data: in datauri"), "parsing attachment")

478 }

479 s = s[len("data:"):]

480 t := strings.SplitN(s, ",", 2)

481 if len(t) != 2 {

482 xcheckuserf(ctx, errors.New("missing comma in datauri"), "parsing attachment")

483 }

484 if !strings.HasSuffix(t[0], "base64") {

485 xcheckuserf(ctx, errors.New("missing base64 in datauri"), "parsing attachment")

486 }

487 ct := strings.TrimSuffix(t[0], "base64")

488 ct = strings.TrimSuffix(ct, ";")

489 if ct == "" {

490 ct = "application/octet-stream"

491 }

492 filename := a.Filename

493 if filename == "" {

494 filename = "unnamed.bin"

495 }

496 params := map[string]string{"name": filename}

497 ct = mime.FormatMediaType(ct, params)

498

499 // Ensure base64 is valid, then we'll write the original string.

500 _, err := io.Copy(io.Discard, base64.NewDecoder(base64.StdEncoding, strings.NewReader(t[1])))

501 xcheckuserf(ctx, err, "parsing attachment as base64")

502

503 xaddAttachmentBase64(ct, filename, []byte(t[1]))

504 }

505

506 if len(m.ForwardAttachments.Paths) > 0 {

507 acc.WithRLock(func() {

508 xdbread(ctx, acc, func(tx *bstore.Tx) {

509 fm := xmessageID(ctx, tx, m.ForwardAttachments.MessageID)

510 msgr := acc.MessageReader(fm)

511 defer func() {

512 err := msgr.Close()

513 log.Check(err, "closing message reader")

514 }()

515

516 fp, err := fm.LoadPart(msgr)

517 xcheckf(ctx, err, "load parsed message")

518

519 for _, path := range m.ForwardAttachments.Paths {

520 ap := fp

521 for _, xp := range path {

522 if xp < 0 || xp >= len(ap.Parts) {

523 xcheckuserf(ctx, errors.New("unknown part"), "looking up attachment")

524 }

525 ap = ap.Parts[xp]

526 }

527

528 filename := tryDecodeParam(log, ap.ContentTypeParams["name"])

529 if filename == "" {

530 filename = "unnamed.bin"

531 }

532 params := map[string]string{"name": filename}

533 if pcharset := ap.ContentTypeParams["charset"]; pcharset != "" {

534 params["charset"] = pcharset

535 }

536 ct := strings.ToLower(ap.MediaType + "/" + ap.MediaSubType)

537 ct = mime.FormatMediaType(ct, params)

538 xaddAttachment(ct, filename, ap.Reader())

539 }

540 })

541 })

542 }

543

544 err = mp.Close()

545 xcheckf(ctx, err, "writing mime multipart")

546 } else {

547 textBody, ct, cte := xc.TextPart(m.TextBody)

548 xc.Header("Content-Type", ct)

549 xc.Header("Content-Transfer-Encoding", cte)

550 xc.Line()

551 xc.Write([]byte(textBody))

552 }

553

554 xc.Flush()

555

556 // Add DKIM-Signature headers.

557 var msgPrefix string

558 fd := fromAddr.Address.Domain

559 confDom, _ := mox.Conf.Domain(fd)

560 if len(confDom.DKIM.Sign) > 0 {

561 dkimHeaders, err := dkim.Sign(ctx, fromAddr.Address.Localpart, fd, confDom.DKIM, smtputf8, dataFile)

562 if err != nil {

563 metricServerErrors.WithLabelValues("dkimsign").Inc()

564 }

565 xcheckf(ctx, err, "sign dkim")

566

567 msgPrefix = dkimHeaders

568 }

569

570 fromPath := smtp.Path{

571 Localpart: fromAddr.Address.Localpart,

572 IPDomain: dns.IPDomain{Domain: fromAddr.Address.Domain},

573 }

574 for _, rcpt := range recipients {

575 rcptMsgPrefix := recvHdrFor(rcpt.Pack(smtputf8)) + msgPrefix

576 msgSize := int64(len(rcptMsgPrefix)) + xc.Size

577 toPath := smtp.Path{

578 Localpart: rcpt.Localpart,

579 IPDomain: dns.IPDomain{Domain: rcpt.Domain},

580 }

581 qm := queue.MakeMsg(reqInfo.AccountName, fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), m.RequireTLS)

582 err := queue.Add(ctx, log, &qm, dataFile)

583 if err != nil {

584 metricSubmission.WithLabelValues("queueerror").Inc()

585 }

586 xcheckf(ctx, err, "adding message to the delivery queue")

587 metricSubmission.WithLabelValues("ok").Inc()

588 }

589

590 var modseq store.ModSeq // Only set if needed.

591

592 // Append message to Sent mailbox and mark original messages as answered/forwarded.

593 acc.WithRLock(func() {

594 var changes []store.Change

595

596 metricked := false

597 defer func() {

598 if x := recover(); x != nil {

599 if !metricked {

600 metricServerErrors.WithLabelValues("submit").Inc()

601 }

602 panic(x)

603 }

604 }()

605 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

606 if m.ResponseMessageID > 0 {

607 rm := xmessageID(ctx, tx, m.ResponseMessageID)

608 oflags := rm.Flags

609 if m.IsForward {

610 rm.Forwarded = true

611 } else {

612 rm.Answered = true

613 }

614 if !rm.Junk && !rm.Notjunk {

615 rm.Notjunk = true

616 }

617 if rm.Flags != oflags {

618 modseq, err = acc.NextModSeq(tx)

619 xcheckf(ctx, err, "next modseq")

620 rm.ModSeq = modseq

621 err := tx.Update(&rm)

622 xcheckf(ctx, err, "updating flags of replied/forwarded message")

623 changes = append(changes, rm.ChangeFlags(oflags))

624

625 err = acc.RetrainMessages(ctx, log, tx, []store.Message{rm}, false)

626 xcheckf(ctx, err, "retraining messages after reply/forward")

627 }

628 }

629

630 sentmb, err := bstore.QueryDB[store.Mailbox](ctx, acc.DB).FilterEqual("Sent", true).Get()

631 if err == bstore.ErrAbsent {

632 // There is no mailbox designated as Sent mailbox, so we're done.

633 return

634 }

635 xcheckf(ctx, err, "message submitted to queue, adding to Sent mailbox")

636

637 if modseq == 0 {

638 modseq, err = acc.NextModSeq(tx)

639 xcheckf(ctx, err, "next modseq")

640 }

641

642 sentm := store.Message{

643 CreateSeq: modseq,

644 ModSeq: modseq,

645 MailboxID: sentmb.ID,

646 MailboxOrigID: sentmb.ID,

647 Flags: store.Flags{Notjunk: true, Seen: true},

648 Size: int64(len(msgPrefix)) + xc.Size,

649 MsgPrefix: []byte(msgPrefix),

650 }

651

652 // Update mailbox before delivery, which changes uidnext.

653 sentmb.Add(sentm.MailboxCounts())

654 err = tx.Update(&sentmb)

655 xcheckf(ctx, err, "updating sent mailbox for counts")

656

657 err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, false, false)

658 if err != nil {

659 metricSubmission.WithLabelValues("storesenterror").Inc()

660 metricked = true

661 }

662 xcheckf(ctx, err, "message submitted to queue, appending message to Sent mailbox")

663

664 changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts())

665 })

666

667 store.BroadcastChanges(acc, changes)

668 })

669}

670

671// MessageMove moves messages to another mailbox. If the message is already in

672// the mailbox an error is returned.

673func (Webmail) MessageMove(ctx context.Context, messageIDs []int64, mailboxID int64) {

674 log := xlog.WithContext(ctx)

675 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

676 acc, err := store.OpenAccount(reqInfo.AccountName)

677 xcheckf(ctx, err, "open account")

678 defer func() {

679 err := acc.Close()

680 log.Check(err, "closing account")

681 }()

682

683 acc.WithRLock(func() {

684 retrain := make([]store.Message, 0, len(messageIDs))

685 removeChanges := map[int64]store.ChangeRemoveUIDs{}

686 // n adds, 1 remove, 2 mailboxcounts, optimistic and at least for a single message.

687 changes := make([]store.Change, 0, len(messageIDs)+3)

688

689 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

690 var mbSrc store.Mailbox

691 var modseq store.ModSeq

692

693 mbDst := xmailboxID(ctx, tx, mailboxID)

694

695 if len(messageIDs) == 0 {

696 return

697 }

698

699 keywords := map[string]struct{}{}

700

701 for _, mid := range messageIDs {

702 m := xmessageID(ctx, tx, mid)

703

704 // We may have loaded this mailbox in the previous iteration of this loop.

705 if m.MailboxID != mbSrc.ID {

706 if mbSrc.ID != 0 {

707 err = tx.Update(&mbSrc)

708 xcheckf(ctx, err, "updating source mailbox counts")

709 changes = append(changes, mbSrc.ChangeCounts())

710 }

711 mbSrc = xmailboxID(ctx, tx, m.MailboxID)

712 }

713

714 if mbSrc.ID == mailboxID {

715 // Client should filter out messages that are already in mailbox.

716 xcheckuserf(ctx, errors.New("already in destination mailbox"), "moving message")

717 }

718

719 if modseq == 0 {

720 modseq, err = acc.NextModSeq(tx)

721 xcheckf(ctx, err, "assigning next modseq")

722 }

723

724 ch := removeChanges[m.MailboxID]

725 ch.UIDs = append(ch.UIDs, m.UID)

726 ch.ModSeq = modseq

727 ch.MailboxID = m.MailboxID

728 removeChanges[m.MailboxID] = ch

729

730 // Copy of message record that we'll insert when UID is freed up.

731 om := m

732 om.PrepareExpunge()

733 om.ID = 0 // Assign new ID.

734 om.ModSeq = modseq

735

736 mbSrc.Sub(m.MailboxCounts())

737

738 if mbDst.Trash {

739 m.Seen = true

740 }

741 conf, _ := acc.Conf()

742 m.MailboxID = mbDst.ID

743 if m.IsReject && m.MailboxDestinedID != 0 {

744 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message

745 // is used for reputation calculation during future deliveries.

746 m.MailboxOrigID = m.MailboxDestinedID

747 m.IsReject = false

748 m.Seen = false

749 }

750 m.UID = mbDst.UIDNext

751 m.ModSeq = modseq

752 mbDst.UIDNext++

753 m.JunkFlagsForMailbox(mbDst, conf)

754 err = tx.Update(&m)

755 xcheckf(ctx, err, "updating moved message in database")

756

757 // Now that UID is unused, we can insert the old record again.

758 err = tx.Insert(&om)

759 xcheckf(ctx, err, "inserting record for expunge after moving message")

760

761 mbDst.Add(m.MailboxCounts())

762

763 changes = append(changes, m.ChangeAddUID())

764 retrain = append(retrain, m)

765

766 for _, kw := range m.Keywords {

767 keywords[kw] = struct{}{}

768 }

769 }

770

771 err = tx.Update(&mbSrc)

772 xcheckf(ctx, err, "updating source mailbox counts")

773

774 changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())

775

776 // Ensure destination mailbox has keywords of the moved messages.

777 var mbKwChanged bool

778 mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))

779 if mbKwChanged {

780 changes = append(changes, mbDst.ChangeKeywords())

781 }

782

783 err = tx.Update(&mbDst)

784 xcheckf(ctx, err, "updating mailbox with uidnext")

785

786 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

787 xcheckf(ctx, err, "retraining messages after move")

788 })

789

790 // Ensure UIDs of the removed message are in increasing order. It is quite common

791 // for all messages to be from a single source mailbox, meaning this is just one

792 // change, for which we preallocated space.

793 for _, ch := range removeChanges {

794 sort.Slice(ch.UIDs, func(i, j int) bool {

795 return ch.UIDs[i] < ch.UIDs[j]

796 })

797 changes = append(changes, ch)

798 }

799 store.BroadcastChanges(acc, changes)

800 })

801}

802

803// MessageDelete permanently deletes messages, without moving them to the Trash mailbox.

804func (Webmail) MessageDelete(ctx context.Context, messageIDs []int64) {

805 log := xlog.WithContext(ctx)

806 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

807 acc, err := store.OpenAccount(reqInfo.AccountName)

808 xcheckf(ctx, err, "open account")

809 defer func() {

810 err := acc.Close()

811 log.Check(err, "closing account")

812 }()

813

814 if len(messageIDs) == 0 {

815 return

816 }

817

818 acc.WithWLock(func() {

819 removeChanges := map[int64]store.ChangeRemoveUIDs{}

820 changes := make([]store.Change, 0, len(messageIDs)+1) // n remove, 1 mailbox counts

821

822 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

823 var modseq store.ModSeq

824 var mb store.Mailbox

825 remove := make([]store.Message, 0, len(messageIDs))

826

827 for _, mid := range messageIDs {

828 m := xmessageID(ctx, tx, mid)

829

830 if m.MailboxID != mb.ID {

831 if mb.ID != 0 {

832 err := tx.Update(&mb)

833 xcheckf(ctx, err, "updating mailbox counts")

834 changes = append(changes, mb.ChangeCounts())

835 }

836 mb = xmailboxID(ctx, tx, m.MailboxID)

837 }

838

839 qmr := bstore.QueryTx[store.Recipient](tx)

840 qmr.FilterEqual("MessageID", m.ID)

841 _, err = qmr.Delete()

842 xcheckf(ctx, err, "removing message recipients")

843

844 mb.Sub(m.MailboxCounts())

845

846 if modseq == 0 {

847 modseq, err = acc.NextModSeq(tx)

848 xcheckf(ctx, err, "assigning next modseq")

849 }

850 m.Expunged = true

851 m.ModSeq = modseq

852 err = tx.Update(&m)

853 xcheckf(ctx, err, "marking message as expunged")

854

855 ch := removeChanges[m.MailboxID]

856 ch.UIDs = append(ch.UIDs, m.UID)

857 ch.MailboxID = m.MailboxID

858 ch.ModSeq = modseq

859 removeChanges[m.MailboxID] = ch

860 remove = append(remove, m)

861 }

862

863 if mb.ID != 0 {

864 err := tx.Update(&mb)

865 xcheckf(ctx, err, "updating count in mailbox")

866 changes = append(changes, mb.ChangeCounts())

867 }

868

869 // Mark removed messages as not needing training, then retrain them, so if they

870 // were trained, they get untrained.

871 for i := range remove {

872 remove[i].Junk = false

873 remove[i].Notjunk = false

874 }

875 err = acc.RetrainMessages(ctx, log, tx, remove, true)

876 xcheckf(ctx, err, "untraining deleted messages")

877 })

878

879 for _, ch := range removeChanges {

880 sort.Slice(ch.UIDs, func(i, j int) bool {

881 return ch.UIDs[i] < ch.UIDs[j]

882 })

883 changes = append(changes, ch)

884 }

885 store.BroadcastChanges(acc, changes)

886 })

887

888 for _, mID := range messageIDs {

889 p := acc.MessagePath(mID)

890 err := os.Remove(p)

891 log.Check(err, "removing message file for expunge")

892 }

893}

894

895// FlagsAdd adds flags, either system flags like \Seen or custom keywords. The

896// flags should be lower-case, but will be converted and verified.

897func (Webmail) FlagsAdd(ctx context.Context, messageIDs []int64, flaglist []string) {

898 log := xlog.WithContext(ctx)

899 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

900 acc, err := store.OpenAccount(reqInfo.AccountName)

901 xcheckf(ctx, err, "open account")

902 defer func() {

903 err := acc.Close()

904 log.Check(err, "closing account")

905 }()

906

907 flags, keywords, err := store.ParseFlagsKeywords(flaglist)

908 xcheckuserf(ctx, err, "parsing flags")

909

910 acc.WithRLock(func() {

911 var changes []store.Change

912

913 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

914 var modseq store.ModSeq

915 var retrain []store.Message

916 var mb, origmb store.Mailbox

917

918 for _, mid := range messageIDs {

919 m := xmessageID(ctx, tx, mid)

920

921 if mb.ID != m.MailboxID {

922 if mb.ID != 0 {

923 err := tx.Update(&mb)

924 xcheckf(ctx, err, "updating mailbox")

925 if mb.MailboxCounts != origmb.MailboxCounts {

926 changes = append(changes, mb.ChangeCounts())

927 }

928 if mb.KeywordsChanged(origmb) {

929 changes = append(changes, mb.ChangeKeywords())

930 }

931 }

932 mb = xmailboxID(ctx, tx, m.MailboxID)

933 origmb = mb

934 }

935 mb.Keywords, _ = store.MergeKeywords(mb.Keywords, keywords)

936

937 mb.Sub(m.MailboxCounts())

938 oflags := m.Flags

939 m.Flags = m.Flags.Set(flags, flags)

940 var kwChanged bool

941 m.Keywords, kwChanged = store.MergeKeywords(m.Keywords, keywords)

942 mb.Add(m.MailboxCounts())

943

944 if m.Flags == oflags && !kwChanged {

945 continue

946 }

947

948 if modseq == 0 {

949 modseq, err = acc.NextModSeq(tx)

950 xcheckf(ctx, err, "assigning next modseq")

951 }

952 m.ModSeq = modseq

953 err = tx.Update(&m)

954 xcheckf(ctx, err, "updating message")

955

956 changes = append(changes, m.ChangeFlags(oflags))

957 retrain = append(retrain, m)

958 }

959

960 if mb.ID != 0 {

961 err := tx.Update(&mb)

962 xcheckf(ctx, err, "updating mailbox")

963 if mb.MailboxCounts != origmb.MailboxCounts {

964 changes = append(changes, mb.ChangeCounts())

965 }

966 if mb.KeywordsChanged(origmb) {

967 changes = append(changes, mb.ChangeKeywords())

968 }

969 }

970

971 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

972 xcheckf(ctx, err, "retraining messages")

973 })

974

975 store.BroadcastChanges(acc, changes)

976 })

977}

978

979// FlagsClear clears flags, either system flags like \Seen or custom keywords.

980func (Webmail) FlagsClear(ctx context.Context, messageIDs []int64, flaglist []string) {

981 log := xlog.WithContext(ctx)

982 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

983 acc, err := store.OpenAccount(reqInfo.AccountName)

984 xcheckf(ctx, err, "open account")

985 defer func() {

986 err := acc.Close()

987 log.Check(err, "closing account")

988 }()

989

990 flags, keywords, err := store.ParseFlagsKeywords(flaglist)

991 xcheckuserf(ctx, err, "parsing flags")

992

993 acc.WithRLock(func() {

994 var retrain []store.Message

995 var changes []store.Change

996

997 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

998 var modseq store.ModSeq

999 var mb, origmb store.Mailbox

1000

1001 for _, mid := range messageIDs {

1002 m := xmessageID(ctx, tx, mid)

1003

1004 if mb.ID != m.MailboxID {

1005 if mb.ID != 0 {

1006 err := tx.Update(&mb)

1007 xcheckf(ctx, err, "updating counts for mailbox")

1008 if mb.MailboxCounts != origmb.MailboxCounts {

1009 changes = append(changes, mb.ChangeCounts())

1010 }

1011 // note: cannot remove keywords from mailbox by removing keywords from message.

1012 }

1013 mb = xmailboxID(ctx, tx, m.MailboxID)

1014 origmb = mb

1015 }

1016

1017 oflags := m.Flags

1018 mb.Sub(m.MailboxCounts())

1019 m.Flags = m.Flags.Set(flags, store.Flags{})

1020 var changed bool

1021 m.Keywords, changed = store.RemoveKeywords(m.Keywords, keywords)

1022 mb.Add(m.MailboxCounts())

1023

1024 if m.Flags == oflags && !changed {

1025 continue

1026 }

1027

1028 if modseq == 0 {

1029 modseq, err = acc.NextModSeq(tx)

1030 xcheckf(ctx, err, "assigning next modseq")

1031 }

1032 m.ModSeq = modseq

1033 err = tx.Update(&m)

1034 xcheckf(ctx, err, "updating message")

1035

1036 changes = append(changes, m.ChangeFlags(oflags))

1037 retrain = append(retrain, m)

1038 }

1039

1040 if mb.ID != 0 {

1041 err := tx.Update(&mb)

1042 xcheckf(ctx, err, "updating keywords in mailbox")

1043 if mb.MailboxCounts != origmb.MailboxCounts {

1044 changes = append(changes, mb.ChangeCounts())

1045 }

1046 // note: cannot remove keywords from mailbox by removing keywords from message.

1047 }

1048

1049 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

1050 xcheckf(ctx, err, "retraining messages")

1051 })

1052

1053 store.BroadcastChanges(acc, changes)

1054 })

1055}

1056

1057// MailboxCreate creates a new mailbox.

1058func (Webmail) MailboxCreate(ctx context.Context, name string) {

1059 log := xlog.WithContext(ctx)

1060 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1061 acc, err := store.OpenAccount(reqInfo.AccountName)

1062 xcheckf(ctx, err, "open account")

1063 defer func() {

1064 err := acc.Close()

1065 log.Check(err, "closing account")

1066 }()

1067

1068 name, _, err = store.CheckMailboxName(name, false)

1069 xcheckuserf(ctx, err, "checking mailbox name")

1070

1071 acc.WithWLock(func() {

1072 var changes []store.Change

1073 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1074 var exists bool

1075 var err error

1076 changes, _, exists, err = acc.MailboxCreate(tx, name)

1077 if exists {

1078 xcheckuserf(ctx, errors.New("mailbox already exists"), "creating mailbox")

1079 }

1080 xcheckf(ctx, err, "creating mailbox")

1081 })

1082

1083 store.BroadcastChanges(acc, changes)

1084 })

1085}

1086

1087// MailboxDelete deletes a mailbox and all its messages.

1088func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {

1089 log := xlog.WithContext(ctx)

1090 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1091 acc, err := store.OpenAccount(reqInfo.AccountName)

1092 xcheckf(ctx, err, "open account")

1093 defer func() {

1094 err := acc.Close()

1095 log.Check(err, "closing account")

1096 }()

1097

1098 // Messages to remove after having broadcasted the removal of messages.

1099 var removeMessageIDs []int64

1100

1101 acc.WithWLock(func() {

1102 var changes []store.Change

1103

1104 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1105 mb := xmailboxID(ctx, tx, mailboxID)

1106 if mb.Name == "Inbox" {

1107 // Inbox is special in IMAP and cannot be removed.

1108 xcheckuserf(ctx, errors.New("cannot remove special Inbox"), "checking mailbox")

1109 }

1110

1111 var hasChildren bool

1112 var err error

1113 changes, removeMessageIDs, hasChildren, err = acc.MailboxDelete(ctx, log, tx, mb)

1114 if hasChildren {

1115 xcheckuserf(ctx, errors.New("mailbox has children"), "deleting mailbox")

1116 }

1117 xcheckf(ctx, err, "deleting mailbox")

1118 })

1119

1120 store.BroadcastChanges(acc, changes)

1121 })

1122

1123 for _, mID := range removeMessageIDs {

1124 p := acc.MessagePath(mID)

1125 err := os.Remove(p)

1126 log.Check(err, "removing message file for mailbox delete", mlog.Field("path", p))

1127 }

1128}

1129

1130// MailboxEmpty empties a mailbox, removing all messages from the mailbox, but not

1131// its child mailboxes.

1132func (Webmail) MailboxEmpty(ctx context.Context, mailboxID int64) {

1133 log := xlog.WithContext(ctx)

1134 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1135 acc, err := store.OpenAccount(reqInfo.AccountName)

1136 xcheckf(ctx, err, "open account")

1137 defer func() {

1138 err := acc.Close()

1139 log.Check(err, "closing account")

1140 }()

1141

1142 var expunged []store.Message

1143

1144 acc.WithWLock(func() {

1145 var changes []store.Change

1146

1147 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1148 mb := xmailboxID(ctx, tx, mailboxID)

1149

1150 modseq, err := acc.NextModSeq(tx)

1151 xcheckf(ctx, err, "next modseq")

1152

1153 // Mark messages as expunged.

1154 qm := bstore.QueryTx[store.Message](tx)

1155 qm.FilterNonzero(store.Message{MailboxID: mb.ID})

1156 qm.FilterEqual("Expunged", false)

1157 qm.SortAsc("UID")

1158 qm.Gather(&expunged)

1159 _, err = qm.UpdateNonzero(store.Message{ModSeq: modseq, Expunged: true})

1160 xcheckf(ctx, err, "deleting messages")

1161

1162 // Remove Recipients.

1163 anyIDs := make([]any, len(expunged))

1164 for i, m := range expunged {

1165 anyIDs[i] = m.ID

1166 }

1167 qmr := bstore.QueryTx[store.Recipient](tx)

1168 qmr.FilterEqual("MessageID", anyIDs...)

1169 _, err = qmr.Delete()

1170 xcheckf(ctx, err, "removing message recipients")

1171

1172 // Adjust mailbox counts, gather UIDs for broadcasted change, prepare for untraining.

1173 uids := make([]store.UID, len(expunged))

1174 for i, m := range expunged {

1175 m.Expunged = false // Gather returns updated values.

1176 mb.Sub(m.MailboxCounts())

1177 uids[i] = m.UID

1178

1179 expunged[i].Junk = false

1180 expunged[i].Notjunk = false

1181 }

1182

1183 err = tx.Update(&mb)

1184 xcheckf(ctx, err, "updating mailbox for counts")

1185

1186 err = acc.RetrainMessages(ctx, log, tx, expunged, true)

1187 xcheckf(ctx, err, "retraining expunged messages")

1188

1189 chremove := store.ChangeRemoveUIDs{MailboxID: mb.ID, UIDs: uids, ModSeq: modseq}

1190 changes = []store.Change{chremove, mb.ChangeCounts()}

1191 })

1192

1193 store.BroadcastChanges(acc, changes)

1194 })

1195

1196 for _, m := range expunged {

1197 p := acc.MessagePath(m.ID)

1198 err := os.Remove(p)

1199 log.Check(err, "removing message file after emptying mailbox", mlog.Field("path", p))

1200 }

1201}

1202

1203// MailboxRename renames a mailbox, possibly moving it to a new parent. The mailbox

1204// ID and its messages are unchanged.

1205func (Webmail) MailboxRename(ctx context.Context, mailboxID int64, newName string) {

1206 log := xlog.WithContext(ctx)

1207 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1208 acc, err := store.OpenAccount(reqInfo.AccountName)

1209 xcheckf(ctx, err, "open account")

1210 defer func() {

1211 err := acc.Close()

1212 log.Check(err, "closing account")

1213 }()

1214

1215 // Renaming Inbox is special for IMAP. For IMAP we have to implement it per the

1216 // standard. We can just say no.

1217 newName, _, err = store.CheckMailboxName(newName, false)

1218 xcheckuserf(ctx, err, "checking new mailbox name")

1219

1220 acc.WithWLock(func() {

1221 var changes []store.Change

1222

1223 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1224 mbsrc := xmailboxID(ctx, tx, mailboxID)

1225 var err error

1226 var isInbox, notExists, alreadyExists bool

1227 changes, isInbox, notExists, alreadyExists, err = acc.MailboxRename(tx, mbsrc, newName)

1228 if isInbox || notExists || alreadyExists {

1229 xcheckuserf(ctx, err, "renaming mailbox")

1230 }

1231 xcheckf(ctx, err, "renaming mailbox")

1232 })

1233

1234 store.BroadcastChanges(acc, changes)

1235 })

1236}

1237

1238// CompleteRecipient returns autocomplete matches for a recipient, returning the

1239// matches, most recently used first, and whether this is the full list and further

1240// requests for longer prefixes aren't necessary.

1241func (Webmail) CompleteRecipient(ctx context.Context, search string) ([]string, bool) {

1242 log := xlog.WithContext(ctx)

1243 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1244 acc, err := store.OpenAccount(reqInfo.AccountName)

1245 xcheckf(ctx, err, "open account")

1246 defer func() {

1247 err := acc.Close()

1248 log.Check(err, "closing account")

1249 }()

1250

1251 search = strings.ToLower(search)

1252

1253 var matches []string

1254 all := true

1255 acc.WithRLock(func() {

1256 xdbread(ctx, acc, func(tx *bstore.Tx) {

1257 type key struct {

1258 localpart smtp.Localpart

1259 domain string

1260 }

1261 seen := map[key]bool{}

1262

1263 q := bstore.QueryTx[store.Recipient](tx)

1264 q.SortDesc("Sent")

1265 err := q.ForEach(func(r store.Recipient) error {

1266 k := key{r.Localpart, r.Domain}

1267 if seen[k] {

1268 return nil

1269 }

1270 // todo: we should have the address including name available in the database for searching. Will result in better matching, and also for the name.

1271 address := fmt.Sprintf("<%s@%s>", r.Localpart.String(), r.Domain)

1272 if !strings.Contains(strings.ToLower(address), search) {

1273 return nil

1274 }

1275 if len(matches) >= 20 {

1276 all = false

1277 return bstore.StopForEach

1278 }

1279

1280 // Look in the message that was sent for a name along with the address.

1281 m := store.Message{ID: r.MessageID}

1282 err := tx.Get(&m)

1283 xcheckf(ctx, err, "get sent message")

1284 if !m.Expunged && m.ParsedBuf != nil {

1285 var part message.Part

1286 err := json.Unmarshal(m.ParsedBuf, &part)

1287 xcheckf(ctx, err, "parsing part")

1288

1289 dom, err := dns.ParseDomain(r.Domain)

1290 xcheckf(ctx, err, "parsing domain of recipient")

1291

1292 var found bool

1293 lp := r.Localpart.String()

1294 checkAddrs := func(l []message.Address) {

1295 if found {

1296 return

1297 }

1298 for _, a := range l {

1299 if a.Name != "" && a.User == lp && strings.EqualFold(a.Host, dom.ASCII) {

1300 found = true

1301 address = addressString(a, false)

1302 return

1303 }

1304 }

1305 }

1306 if part.Envelope != nil {

1307 env := part.Envelope

1308 checkAddrs(env.To)

1309 checkAddrs(env.CC)

1310 checkAddrs(env.BCC)

1311 }

1312 }

1313

1314 matches = append(matches, address)

1315 seen[k] = true

1316 return nil

1317 })

1318 xcheckf(ctx, err, "listing recipients")

1319 })

1320 })

1321 return matches, all

1322}

1323

1324// addressString returns an address into a string as it could be used in a message header.

1325func addressString(a message.Address, smtputf8 bool) string {

1326 host := a.Host

1327 dom, err := dns.ParseDomain(a.Host)

1328 if err == nil {

1329 if smtputf8 && dom.Unicode != "" {

1330 host = dom.Unicode

1331 } else {

1332 host = dom.ASCII

1333 }

1334 }

1335 s := "<" + a.User + "@" + host + ">"

1336 if a.Name != "" {

1337 // todo: properly encoded/escaped name

1338 s = a.Name + " " + s

1339 }

1340 return s

1341}

1342

1343// MailboxSetSpecialUse sets the special use flags of a mailbox.

1344func (Webmail) MailboxSetSpecialUse(ctx context.Context, mb store.Mailbox) {

1345 log := xlog.WithContext(ctx)

1346 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1347 acc, err := store.OpenAccount(reqInfo.AccountName)

1348 xcheckf(ctx, err, "open account")

1349 defer func() {

1350 err := acc.Close()

1351 log.Check(err, "closing account")

1352 }()

1353

1354 acc.WithWLock(func() {

1355 var changes []store.Change

1356

1357 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1358 xmb := xmailboxID(ctx, tx, mb.ID)

1359

1360 // We only allow a single mailbox for each flag (JMAP requirement). So for any flag

1361 // we set, we clear it for the mailbox(es) that had it, if any.

1362 clearPrevious := func(clear bool, specialUse string) {

1363 if !clear {

1364 return

1365 }

1366 var ombl []store.Mailbox

1367 q := bstore.QueryTx[store.Mailbox](tx)

1368 q.FilterNotEqual("ID", mb.ID)

1369 q.FilterEqual(specialUse, true)

1370 q.Gather(&ombl)

1371 _, err := q.UpdateField(specialUse, false)

1372 xcheckf(ctx, err, "updating previous special-use mailboxes")

1373

1374 for _, omb := range ombl {

1375 changes = append(changes, omb.ChangeSpecialUse())

1376 }

1377 }

1378 clearPrevious(mb.Archive, "Archive")

1379 clearPrevious(mb.Draft, "Draft")

1380 clearPrevious(mb.Junk, "Junk")

1381 clearPrevious(mb.Sent, "Sent")

1382 clearPrevious(mb.Trash, "Trash")

1383

1384 xmb.SpecialUse = mb.SpecialUse

1385 err = tx.Update(&xmb)

1386 xcheckf(ctx, err, "updating special-use flags for mailbox")

1387 changes = append(changes, xmb.ChangeSpecialUse())

1388 })

1389

1390 store.BroadcastChanges(acc, changes)

1391 })

1392}

1393

1394// ThreadCollapse saves the ThreadCollapse field for the messages and its

1395// children. The messageIDs are typically thread roots. But not all roots

1396// (without parent) of a thread need to have the same collapsed state.

1397func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse bool) {

1398 log := xlog.WithContext(ctx)

1399 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1400 acc, err := store.OpenAccount(reqInfo.AccountName)

1401 xcheckf(ctx, err, "open account")

1402 defer func() {

1403 err := acc.Close()

1404 log.Check(err, "closing account")

1405 }()

1406

1407 if len(messageIDs) == 0 {

1408 xcheckuserf(ctx, errors.New("no messages"), "setting collapse")

1409 }

1410

1411 acc.WithWLock(func() {

1412 changes := make([]store.Change, 0, len(messageIDs))

1413 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1414 // Gather ThreadIDs to list all potential messages, for a way to get all potential

1415 // (child) messages. Further refined in FilterFn.

1416 threadIDs := map[int64]struct{}{}

1417 msgIDs := map[int64]struct{}{}

1418 for _, id := range messageIDs {

1419 m := store.Message{ID: id}

1420 err := tx.Get(&m)

1421 if err == bstore.ErrAbsent {

1422 xcheckuserf(ctx, err, "get message")

1423 }

1424 xcheckf(ctx, err, "get message")

1425 threadIDs[m.ThreadID] = struct{}{}

1426 msgIDs[id] = struct{}{}

1427 }

1428

1429 var updated []store.Message

1430 q := bstore.QueryTx[store.Message](tx)

1431 q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)

1432 q.FilterNotEqual("ThreadCollapsed", collapse)

1433 q.FilterFn(func(tm store.Message) bool {

1434 for _, id := range tm.ThreadParentIDs {

1435 if _, ok := msgIDs[id]; ok {

1436 return true

1437 }

1438 }

1439 _, ok := msgIDs[tm.ID]

1440 return ok

1441 })

1442 q.Gather(&updated)

1443 q.SortAsc("ID") // Consistent order for testing.

1444 _, err = q.UpdateFields(map[string]any{"ThreadCollapsed": collapse})

1445 xcheckf(ctx, err, "updating collapse in database")

1446

1447 for _, m := range updated {

1448 changes = append(changes, m.ChangeThread())

1449 }

1450 })

1451 store.BroadcastChanges(acc, changes)

1452 })

1453}

1454

1455// ThreadMute saves the ThreadMute field for the messages and their children.

1456// If messages are muted, they are also marked collapsed.

1457func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) {

1458 log := xlog.WithContext(ctx)

1459 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1460 acc, err := store.OpenAccount(reqInfo.AccountName)

1461 xcheckf(ctx, err, "open account")

1462 defer func() {

1463 err := acc.Close()

1464 log.Check(err, "closing account")

1465 }()

1466

1467 if len(messageIDs) == 0 {

1468 xcheckuserf(ctx, errors.New("no messages"), "setting mute")

1469 }

1470

1471 acc.WithWLock(func() {

1472 changes := make([]store.Change, 0, len(messageIDs))

1473 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1474 threadIDs := map[int64]struct{}{}

1475 msgIDs := map[int64]struct{}{}

1476 for _, id := range messageIDs {

1477 m := store.Message{ID: id}

1478 err := tx.Get(&m)

1479 if err == bstore.ErrAbsent {

1480 xcheckuserf(ctx, err, "get message")

1481 }

1482 xcheckf(ctx, err, "get message")

1483 threadIDs[m.ThreadID] = struct{}{}

1484 msgIDs[id] = struct{}{}

1485 }

1486

1487 var updated []store.Message

1488

1489 q := bstore.QueryTx[store.Message](tx)

1490 q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)

1491 q.FilterFn(func(tm store.Message) bool {

1492 if tm.ThreadMuted == mute && (!mute || tm.ThreadCollapsed) {

1493 return false

1494 }

1495 for _, id := range tm.ThreadParentIDs {

1496 if _, ok := msgIDs[id]; ok {

1497 return true

1498 }

1499 }

1500 _, ok := msgIDs[tm.ID]

1501 return ok

1502 })

1503 q.Gather(&updated)

1504 fields := map[string]any{"ThreadMuted": mute}

1505 if mute {

1506 fields["ThreadCollapsed"] = true

1507 }

1508 _, err = q.UpdateFields(fields)

1509 xcheckf(ctx, err, "updating mute in database")

1510

1511 for _, m := range updated {

1512 changes = append(changes, m.ChangeThread())

1513 }

1514 })

1515 store.BroadcastChanges(acc, changes)

1516 })

1517}

1518

1519// SecurityResult indicates whether a security feature is supported.

1520type SecurityResult string

1521

1522const (

1523 SecurityResultError SecurityResult = "error"

1524 SecurityResultNo SecurityResult = "no"

1525 SecurityResultYes SecurityResult = "yes"

1526 // Unknown whether supported. Finding out may only be (reasonably) possible when

1527 // trying (e.g. SMTP STARTTLS). Once tried, the result may be cached for future

1528 // lookups.

1529 SecurityResultUnknown SecurityResult = "unknown"

1530)

1531

1532// RecipientSecurity is a quick analysis of the security properties of delivery to

1533// the recipient (domain).

1534type RecipientSecurity struct {

1535 // Whether recipient domain supports (opportunistic) STARTTLS, as seen during most

1536 // recent delivery attempt. Will be "unknown" if no delivery to the domain has been

1537 // attempted yet.

1538 STARTTLS SecurityResult

1539

1540 // Whether we have a stored enforced MTA-STS policy, or domain has MTA-STS DNS

1541 // record.

1542 MTASTS SecurityResult

1543

1544 // Whether MX lookup response was DNSSEC-signed.

1545 DNSSEC SecurityResult

1546

1547 // Whether first delivery destination has DANE records.

1548 DANE SecurityResult

1549

1550 // Whether recipient domain is known to implement the REQUIRETLS SMTP extension.

1551 // Will be "unknown" if no delivery to the domain has been attempted yet.

1552 RequireTLS SecurityResult

1553}

1554

1555// RecipientSecurity looks up security properties of the address in the

1556// single-address message addressee (as it appears in a To/Cc/Bcc/etc header).

1557func (Webmail) RecipientSecurity(ctx context.Context, messageAddressee string) (RecipientSecurity, error) {

1558 resolver := dns.StrictResolver{Pkg: "webmail"}

1559 return recipientSecurity(ctx, resolver, messageAddressee)

1560}

1561

1562// logPanic can be called with a defer from a goroutine to prevent the entire program from being shutdown in case of a panic.

1563func logPanic(ctx context.Context) {

1564 x := recover()

1565 if x == nil {

1566 return

1567 }

1568 log := xlog.WithContext(ctx)

1569 log.Error("recover from panic", mlog.Field("panic", x))

1570 debug.PrintStack()

1571 metrics.PanicInc(metrics.Webmail)

1572}

1573

1574// separate function for testing with mocked resolver.

1575func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddressee string) (RecipientSecurity, error) {

1576 log := xlog.WithContext(ctx)

1577

1578 rs := RecipientSecurity{

1579 SecurityResultUnknown,

1580 SecurityResultUnknown,

1581 SecurityResultUnknown,

1582 SecurityResultUnknown,

1583 SecurityResultUnknown,

1584 }

1585

1586 msgAddr, err := mail.ParseAddress(messageAddressee)

1587 if err != nil {

1588 return rs, fmt.Errorf("parsing message addressee: %v", err)

1589 }

1590

1591 addr, err := smtp.ParseAddress(msgAddr.Address)

1592 if err != nil {

1593 return rs, fmt.Errorf("parsing address: %v", err)

1594 }

1595

1596 var wg sync.WaitGroup

1597

1598 // MTA-STS.

1599 wg.Add(1)

1600 go func() {

1601 defer logPanic(ctx)

1602 defer wg.Done()

1603

1604 policy, _, _, err := mtastsdb.Get(ctx, resolver, addr.Domain)

1605 if policy != nil && policy.Mode == mtasts.ModeEnforce {

1606 rs.MTASTS = SecurityResultYes

1607 } else if err == nil {

1608 rs.MTASTS = SecurityResultNo

1609 } else {

1610 rs.MTASTS = SecurityResultError

1611 }

1612 }()

1613

1614 // DNSSEC and DANE.

1615 wg.Add(1)

1616 go func() {

1617 defer logPanic(ctx)

1618 defer wg.Done()

1619

1620 _, origNextHopAuthentic, expandedNextHopAuthentic, _, hosts, _, err := smtpclient.GatherDestinations(ctx, log, resolver, dns.IPDomain{Domain: addr.Domain})

1621 if err != nil {

1622 rs.DNSSEC = SecurityResultError

1623 return

1624 }

1625 if origNextHopAuthentic && expandedNextHopAuthentic {

1626 rs.DNSSEC = SecurityResultYes

1627 } else {

1628 rs.DNSSEC = SecurityResultNo

1629 }

1630

1631 if !origNextHopAuthentic {

1632 rs.DANE = SecurityResultNo

1633 return

1634 }

1635

1636 // We're only looking at the first host to deliver to (typically first mx destination).

1637 if len(hosts) == 0 || hosts[0].Domain.IsZero() {

1638 return // Should not happen.

1639 }

1640 host := hosts[0]

1641

1642 // Resolve the IPs. Required for DANE to prevent bad DNS servers from causing an

1643 // error result instead of no-DANE result.

1644 authentic, expandedAuthentic, expandedHost, _, _, err := smtpclient.GatherIPs(ctx, log, resolver, host, map[string][]net.IP{})

1645 if err != nil {

1646 rs.DANE = SecurityResultError

1647 return

1648 }

1649 if !authentic {

1650 rs.DANE = SecurityResultNo

1651 return

1652 }

1653

1654 daneRequired, _, _, err := smtpclient.GatherTLSA(ctx, log, resolver, host.Domain, expandedAuthentic, expandedHost)

1655 if err != nil {

1656 rs.DANE = SecurityResultError

1657 return

1658 } else if daneRequired {

1659 rs.DANE = SecurityResultYes

1660 } else {

1661 rs.DANE = SecurityResultNo

1662 }

1663 }()

1664

1665 // STARTTLS and RequireTLS

1666 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1667 acc, err := store.OpenAccount(reqInfo.AccountName)

1668 xcheckf(ctx, err, "open account")

1669 defer func() {

1670 if acc != nil {

1671 err := acc.Close()

1672 log.Check(err, "closing account")

1673 }

1674 }()

1675

1676 err = acc.DB.Read(ctx, func(tx *bstore.Tx) error {

1677 q := bstore.QueryTx[store.RecipientDomainTLS](tx)

1678 q.FilterNonzero(store.RecipientDomainTLS{Domain: addr.Domain.Name()})

1679 rd, err := q.Get()

1680 if err == bstore.ErrAbsent {

1681 return nil

1682 } else if err != nil {

1683 rs.STARTTLS = SecurityResultError

1684 rs.RequireTLS = SecurityResultError

1685 log.Errorx("looking up recipient domain", err, mlog.Field("domain", addr.Domain))

1686 return nil

1687 }

1688 if rd.STARTTLS {

1689 rs.STARTTLS = SecurityResultYes

1690 } else {

1691 rs.STARTTLS = SecurityResultNo

1692 }

1693 if rd.RequireTLS {

1694 rs.RequireTLS = SecurityResultYes

1695 } else {

1696 rs.RequireTLS = SecurityResultNo

1697 }

1698 return nil

1699 })

1700 xcheckf(ctx, err, "lookup recipient domain")

1701

1702 // Close account as soon as possible, not after waiting for MTA-STS/DNSSEC/DANE

1703 // checks to complete, which can take a while.

1704 err = acc.Close()

1705 log.Check(err, "closing account")

1706 acc = nil

1707

1708 wg.Wait()

1709

1710 return rs, nil

1711}

1712

1713func slicesAny[T any](l []T) []any {

1714 r := make([]any, len(l))

1715 for i, v := range l {

1716 r[i] = v

1717 }

1718 return r

1719}

1720

1721// SSETypes exists to ensure the generated API contains the types, for use in SSE events.

1722func (Webmail) SSETypes() (start EventStart, viewErr EventViewErr, viewReset EventViewReset, viewMsgs EventViewMsgs, viewChanges EventViewChanges, msgAdd ChangeMsgAdd, msgRemove ChangeMsgRemove, msgFlags ChangeMsgFlags, msgThread ChangeMsgThread, mailboxRemove ChangeMailboxRemove, mailboxAdd ChangeMailboxAdd, mailboxRename ChangeMailboxRename, mailboxCounts ChangeMailboxCounts, mailboxSpecialUse ChangeMailboxSpecialUse, mailboxKeywords ChangeMailboxKeywords, flags store.Flags) {

1723 return

1724}

1725