1// Package imapserver implements an IMAPv4 server, rev2 (RFC 9051) and rev1 with extensions (RFC 3501 and more).
7IMAP4rev2 includes functionality that was in extensions for IMAP4rev1. The
8extensions sometimes include features not in IMAP4rev2. We want IMAP4rev1-only
9implementations to use extensions, so we implement the full feature set of the
10extension and announce it as capability. The extensions: LITERAL+, IDLE,
11NAMESPACE, BINARY, UNSELECT, UIDPLUS, ESEARCH, SEARCHRES, SASL-IR, ENABLE,
12LIST-EXTENDED, SPECIAL-USE, MOVE, UTF8=ONLY.
14We take a liberty with UTF8=ONLY. We are supposed to wait for ENABLE of
15UTF8=ACCEPT or IMAP4rev2 before we respond with quoted strings that contain
16non-ASCII UTF-8. Until that's enabled, we do use UTF-7 for mailbox names. See
19- We never execute multiple commands at the same time for a connection. We expect a client to open multiple connections instead.
../rfc/9051:1110
20- Do not write output on a connection with an account lock held. Writing can block, a slow client could block account operations.
21- When handling commands that modify the selected mailbox, always check that the mailbox is not opened readonly. And always revalidate the selected mailbox, another session may have deleted the mailbox.
22- After making changes to an account/mailbox/message, you must broadcast changes. You must do this with the account lock held. Otherwise, other later changes (e.g. message deliveries) may be made and broadcast before changes that were made earlier. Make sure to commit changes in the database first, because the commit may fail.
23- Mailbox hierarchies are slash separated, no leading slash. We keep the case, except INBOX is renamed to Inbox, also for submailboxes in INBOX. We don't allow existence of a child where its parent does not exist. We have no \NoInferiors or \NoSelect. Newly created mailboxes are automatically subscribed.
24- For CONDSTORE and QRESYNC support, we set "modseq" for each change/expunge. Once expunged, a modseq doesn't change anymore. We don't yet remove old expunged records. The records aren't too big. Next step may be to let an admin reclaim space manually.
28- todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64?
29- todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command.
62 "golang.org/x/text/unicode/norm"
64 "github.com/prometheus/client_golang/prometheus"
65 "github.com/prometheus/client_golang/prometheus/promauto"
67 "github.com/mjl-/bstore"
68 "github.com/mjl-/flate"
70 "github.com/mjl-/mox/config"
71 "github.com/mjl-/mox/junk"
72 "github.com/mjl-/mox/message"
73 "github.com/mjl-/mox/metrics"
74 "github.com/mjl-/mox/mlog"
75 "github.com/mjl-/mox/mox-"
76 "github.com/mjl-/mox/moxio"
77 "github.com/mjl-/mox/moxvar"
78 "github.com/mjl-/mox/ratelimit"
79 "github.com/mjl-/mox/scram"
80 "github.com/mjl-/mox/store"
84 metricIMAPConnection = promauto.NewCounterVec(
85 prometheus.CounterOpts{
86 Name: "mox_imap_connection_total",
87 Help: "Incoming IMAP connections.",
90 "service", // imap, imaps
93 metricIMAPCommands = promauto.NewHistogramVec(
94 prometheus.HistogramOpts{
95 Name: "mox_imap_command_duration_seconds",
96 Help: "IMAP command duration and result codes in seconds.",
97 Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
101 "result", // ok, panic, ioerror, badsyntax, servererror, usererror, error
106var unhandledPanics atomic.Int64 // For tests.
108var limiterConnectionrate, limiterConnections *ratelimit.Limiter
111 // Also called by tests, so they don't trigger the rate limiter.
117 limiterConnectionrate = &ratelimit.Limiter{
118 WindowLimits: []ratelimit.WindowLimit{
121 Limits: [...]int64{300, 900, 2700},
125 limiterConnections = &ratelimit.Limiter{
126 WindowLimits: []ratelimit.WindowLimit{
128 Window: time.Duration(math.MaxInt64), // All of time.
129 Limits: [...]int64{30, 90, 270},
135// Delay after bad/suspicious behaviour. Tests set these to zero.
136var badClientDelay = time.Second // Before reads and after 1-byte writes for probably spammers.
137var authFailDelay = time.Second // After authentication failure.
139// Capabilities (extensions) the server supports. Connections will add a few more,
140// e.g. STARTTLS, LOGINDISABLED, AUTH=PLAIN. Accounts may disable some
141// capabilities, to work around compatibility issues.
143// We always announce support for SCRAM PLUS-variants, also on connections without
144// TLS. The client should not be selecting PLUS variants on non-TLS connections,
145// instead opting to do the bare SCRAM variant without indicating the server claims
146// to support the PLUS variant (skipping the server downgrade detection check).
147var serverCapabilitiesList = []string{
163 "CREATE-SPECIAL-USE", //
166 "AUTH=SCRAM-SHA-256", //
168 "AUTH=SCRAM-SHA-1", //
171 "APPENDLIMIT=9223372036854775807", //
../rfc/7889:129, we support the max possible size, 1<<63 - 1
176 "QUOTA=RES-STORAGE", //
189 // "COMPRESS=DEFLATE", //
../rfc/4978, disabled for interoperability issues: The flate reader (inflate) still blocks on partial flushes, preventing progress.
191var serverCapabilities = strings.Join(serverCapabilitiesList, " ")
197 connBroken bool // Once broken, we won't flush any more data.
198 tls bool // Whether TLS has been initialized.
199 viaHTTPS bool // Whether this connection came in via HTTPS (using TLS ALPN).
201 br *bufio.Reader // From remote, with TLS unwrapped in case of TLS, and possibly wrapping inflate.
202 tr *moxio.TraceReader // Kept to change trace level when reading/writing cmd/auth/data.
203 line chan lineErr // If set, instead of reading from br, a line is read from this channel. For reading a line in IDLE while also waiting for mailbox/account updates.
204 lastLine string // For detecting if syntax error is fatal, i.e. if this ends with a literal. Without crlf.
205 xbw *bufio.Writer // To remote, with TLS added in case of TLS, and possibly wrapping deflate, see conn.xflateWriter. Writes go through xtw to conn.Write, which panics on errors, hence the "x".
206 xtw *moxio.TraceWriter
207 xflateWriter *moxio.FlateWriter // For flushing output after flushing conn.xbw, and for closing.
208 xflateBW *bufio.Writer // Wraps raw connection writes, xflateWriter writes here, also needs flushing.
209 slow bool // If set, reads are done with a 1 second sleep, and writes are done 1 byte at a time, to keep spammers busy.
210 lastlog time.Time // For printing time since previous log line.
211 baseTLSConfig *tls.Config // Base TLS config to use for handshake.
213 noRequireSTARTTLS bool
214 cmd string // Currently executing, for deciding to xapplyChanges and logging.
215 cmdMetric string // Currently executing, for metrics.
217 ncmds int // Number of commands processed. Used to abort connection when first incoming command is unknown/invalid.
218 log mlog.Log // Used for all synchronous logging on this connection, see logbg for logging in a separate goroutine.
219 enabled map[capability]bool // All upper-case.
220 compress bool // Whether compression is enabled, via compress command.
221 notify *notify // For the NOTIFY extension. Event/change filtering active if non-nil.
223 // Set by SEARCH with SAVE. Can be used by commands accepting a sequence-set with
224 // value "$". When used, UIDs must be verified to still exist, because they may
225 // have been expunged. Cleared by a SELECT or EXAMINE.
226 // Nil means no searchResult is present. An empty list is a valid searchResult,
227 // just not matching any messages.
229 searchResult []store.UID
231 // userAgent is set by the ID command, which can happen at any time (before or
232 // after the authentication attempt we want to log it with).
234 // loginAttempt is set during authentication, typically picked up by the ID command
235 // that soon follows, or it will be flushed within 1s, or on connection teardown.
236 loginAttempt *store.LoginAttempt
237 loginAttemptTime time.Time
239 // Only set when connection has been authenticated. These can be set even when
240 // c.state is stateNotAuthenticated, for TLS client certificate authentication. In
241 // that case, credentials aren't used until the authentication command with the
242 // SASL "EXTERNAL" mechanism.
243 authFailed int // Number of failed auth attempts. For slowing down remote with many failures.
244 noPreauth bool // If set, don't switch connection to "authenticated" after TLS handshake with client certificate authentication.
245 username string // Full username as used during login.
246 account *store.Account
247 comm *store.Comm // For sending/receiving changes on mailboxes in account, e.g. from messages incoming on smtp, or another imap client.
249 mailboxID int64 // Only for StateSelected.
250 readonly bool // If opened mailbox is readonly.
251 uidonly bool // If uidonly is enabled, uids is empty and cannot be used.
252 uidnext store.UID // We don't return search/fetch/etc results for uids >= uidnext, which is updated when applying changes.
253 exists uint32 // Needed for uidonly, equal to len(uids) for non-uidonly sessions.
254 uids []store.UID // UIDs known in this session, sorted. todo future: store more space-efficiently, as ranges.
257// capability for use with ENABLED and CAPABILITY. We always keep this upper case,
258// e.g. IMAP4REV2. These values are treated case-insensitive, but it's easier for
259// comparison to just always have the same case.
260type capability string
263 capIMAP4rev2 capability = "IMAP4REV2"
264 capUTF8Accept capability = "UTF8=ACCEPT"
265 capCondstore capability = "CONDSTORE"
266 capQresync capability = "QRESYNC"
267 capMetadata capability = "METADATA"
268 capUIDOnly capability = "UIDONLY"
279 stateNotAuthenticated state = iota
284func stateCommands(cmds ...string) map[string]struct{} {
285 r := map[string]struct{}{}
286 for _, cmd := range cmds {
293 commandsStateAny = stateCommands("capability", "noop", "logout", "id")
294 commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login")
295 commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata", "compress", "esearch", "notify")
296 commandsStateSelected = stateCommands("check", "close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move", "replace", "uid replace", "esearch")
299// Commands that use sequence numbers. Cannot be used when UIDONLY is enabled.
300// Commands like UID SEARCH have additional checks for some parameters.
301var commandsSequence = stateCommands("search", "fetch", "store", "copy", "move", "replace")
303var commands = map[string]func(c *conn, tag, cmd string, p *parser){
305 "capability": (*conn).cmdCapability,
306 "noop": (*conn).cmdNoop,
307 "logout": (*conn).cmdLogout,
311 "starttls": (*conn).cmdStarttls,
312 "authenticate": (*conn).cmdAuthenticate,
313 "login": (*conn).cmdLogin,
315 // Authenticated and selected.
316 "enable": (*conn).cmdEnable,
317 "select": (*conn).cmdSelect,
318 "examine": (*conn).cmdExamine,
319 "create": (*conn).cmdCreate,
320 "delete": (*conn).cmdDelete,
321 "rename": (*conn).cmdRename,
322 "subscribe": (*conn).cmdSubscribe,
323 "unsubscribe": (*conn).cmdUnsubscribe,
324 "list": (*conn).cmdList,
325 "lsub": (*conn).cmdLsub,
326 "namespace": (*conn).cmdNamespace,
327 "status": (*conn).cmdStatus,
328 "append": (*conn).cmdAppend,
329 "idle": (*conn).cmdIdle,
330 "getquotaroot": (*conn).cmdGetquotaroot,
331 "getquota": (*conn).cmdGetquota,
332 "getmetadata": (*conn).cmdGetmetadata,
333 "setmetadata": (*conn).cmdSetmetadata,
334 "compress": (*conn).cmdCompress,
335 "esearch": (*conn).cmdEsearch,
339 "check": (*conn).cmdCheck,
340 "close": (*conn).cmdClose,
341 "unselect": (*conn).cmdUnselect,
342 "expunge": (*conn).cmdExpunge,
343 "uid expunge": (*conn).cmdUIDExpunge,
344 "search": (*conn).cmdSearch,
345 "uid search": (*conn).cmdUIDSearch,
346 "fetch": (*conn).cmdFetch,
347 "uid fetch": (*conn).cmdUIDFetch,
348 "store": (*conn).cmdStore,
349 "uid store": (*conn).cmdUIDStore,
350 "copy": (*conn).cmdCopy,
351 "uid copy": (*conn).cmdUIDCopy,
352 "move": (*conn).cmdMove,
353 "uid move": (*conn).cmdUIDMove,
355 "replace": (*conn).cmdReplace,
356 "uid replace": (*conn).cmdUIDReplace,
359var errIO = errors.New("io error") // For read/write errors and errors that should close the connection.
360var errProtocol = errors.New("protocol error") // For protocol errors for which a stack trace should be printed.
364// check err for sanity.
365// if not nil and checkSanity true (set during tests), then panic. if not nil during normal operation, just log.
366func (c *conn) xsanity(err error, format string, args ...any) {
371 panic(fmt.Errorf("%s: %s", fmt.Sprintf(format, args...), err))
373 c.log.Errorx(fmt.Sprintf(format, args...), err)
376func (c *conn) xbrokenf(format string, args ...any) {
378 panic(fmt.Errorf(format, args...))
383// Listen initializes all imap listeners for the configuration, and stores them for Serve to start them.
385 names := slices.Sorted(maps.Keys(mox.Conf.Static.Listeners))
386 for _, name := range names {
387 listener := mox.Conf.Static.Listeners[name]
389 var tlsConfig *tls.Config
390 var noTLSClientAuth bool
391 if listener.TLS != nil {
392 tlsConfig = listener.TLS.Config
393 noTLSClientAuth = listener.TLS.ClientAuthDisabled
396 if listener.IMAP.Enabled {
397 port := config.Port(listener.IMAP.Port, 143)
398 for _, ip := range listener.IPs {
399 listen1("imap", name, ip, port, tlsConfig, false, noTLSClientAuth, listener.IMAP.NoRequireSTARTTLS)
403 if listener.IMAPS.Enabled {
404 port := config.Port(listener.IMAPS.Port, 993)
405 for _, ip := range listener.IPs {
406 listen1("imaps", name, ip, port, tlsConfig, true, noTLSClientAuth, false)
414func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config, xtls, noTLSClientAuth, noRequireSTARTTLS bool) {
415 log := mlog.New("imapserver", nil)
416 addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
417 if os.Getuid() == 0 {
418 log.Print("listening for imap",
419 slog.String("listener", listenerName),
420 slog.String("addr", addr),
421 slog.String("protocol", protocol))
423 network := mox.Network(ip)
424 ln, err := mox.Listen(network, addr)
426 log.Fatalx("imap: listen for imap", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
429 // Each listener gets its own copy of the config, so session keys between different
430 // ports on same listener aren't shared. We rotate session keys explicitly in this
431 // base TLS config because each connection clones the TLS config before using. The
432 // base TLS config would never get automatically managed/rotated session keys.
433 if tlsConfig != nil {
434 tlsConfig = tlsConfig.Clone()
435 mox.StartTLSSessionTicketKeyRefresher(mox.Shutdown, log, tlsConfig)
440 conn, err := ln.Accept()
442 log.Infox("imap: accept", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
446 metricIMAPConnection.WithLabelValues(protocol).Inc()
447 go serve(listenerName, mox.Cid(), tlsConfig, conn, xtls, noTLSClientAuth, noRequireSTARTTLS, false, "")
451 servers = append(servers, serve)
454// ServeTLSConn serves IMAP on a TLS connection.
455func ServeTLSConn(listenerName string, conn *tls.Conn, tlsConfig *tls.Config) {
456 serve(listenerName, mox.Cid(), tlsConfig, conn, true, true, false, true, "")
459func ServeConnPreauth(listenerName string, cid int64, conn net.Conn, preauthAddress string) {
460 serve(listenerName, cid, nil, conn, false, true, true, false, preauthAddress)
463// Serve starts serving on all listeners, launching a goroutine per listener.
465 for _, serve := range servers {
471// Logbg returns a logger for logging in the background (in a goroutine), eg for
472// logging LoginAttempts. The regular c.log has a handler that evaluates fields on
473// the connection at time of logging, which may happen at the same time as
474// modifications to those fields.
475func (c *conn) logbg() mlog.Log {
476 log := mlog.New("imapserver", nil).WithCid(c.cid)
477 if c.username != "" {
478 log = log.With(slog.String("username", c.username))
483// returns whether this connection accepts utf-8 in strings.
484func (c *conn) utf8strings() bool {
485 return c.enabled[capIMAP4rev2] || c.enabled[capUTF8Accept]
488func (c *conn) xdbwrite(fn func(tx *bstore.Tx)) {
489 err := c.account.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
493 xcheckf(err, "transaction")
496func (c *conn) xdbread(fn func(tx *bstore.Tx)) {
497 err := c.account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
501 xcheckf(err, "transaction")
504// Closes the currently selected/active mailbox, setting state from selected to authenticated.
505// Does not remove messages marked for deletion.
506func (c *conn) unselect() {
507 // Flush any pending delayed changes as if the mailbox is still selected. Probably
508 // better than causing STATUS responses for the mailbox being unselected but which
509 // is still selected.
510 c.flushNotifyDelayed()
512 if c.state == stateSelected {
513 c.state = stateAuthenticated
521func (c *conn) flushNotifyDelayed() {
525 delayed := c.notify.Delayed
526 c.notify.Delayed = nil
527 c.flushChanges(delayed)
530// flushChanges is called for NOTIFY changes we shouldn't send untagged messages
531// about but must process for message removals. We don't update the selected
532// mailbox message sequence numbers, since the client would have no idea we
533// adjusted message sequence numbers. Combined with NOTIFY NONE, this means
534// messages may be erased that the client thinks still exists in its session.
535func (c *conn) flushChanges(changes []store.Change) {
536 for _, change := range changes {
537 switch ch := change.(type) {
538 case store.ChangeRemoveUIDs:
539 c.comm.RemovalSeen(ch)
544func (c *conn) setSlow(on bool) {
546 c.log.Debug("connection changed to slow")
547 } else if !on && c.slow {
548 c.log.Debug("connection restored to regular pace")
553// Write makes a connection an io.Writer. It panics for i/o errors. These errors
554// are handled in the connection command loop.
555func (c *conn) Write(buf []byte) (int, error) {
563 err := c.conn.SetWriteDeadline(time.Now().Add(30 * time.Second))
564 c.log.Check(err, "setting write deadline")
566 nn, err := c.conn.Write(buf[:chunk])
568 c.xbrokenf("write: %s (%w)", err, errIO)
572 if len(buf) > 0 && badClientDelay > 0 {
573 mox.Sleep(mox.Context, badClientDelay)
579func (c *conn) xtraceread(level slog.Level) func() {
582 c.tr.SetTrace(mlog.LevelTrace)
586func (c *conn) xtracewrite(level slog.Level) func() {
588 c.xtw.SetTrace(level)
591 c.xtw.SetTrace(mlog.LevelTrace)
595// Cache of line buffers for reading commands.
597var bufpool = moxio.NewBufpool(8, 16*1024)
599// read line from connection, not going through line channel.
600func (c *conn) readline0() (string, error) {
601 if c.slow && badClientDelay > 0 {
602 mox.Sleep(mox.Context, badClientDelay)
605 d := 30 * time.Minute
606 if c.state == stateNotAuthenticated {
609 err := c.conn.SetReadDeadline(time.Now().Add(d))
610 c.log.Check(err, "setting read deadline")
612 line, err := bufpool.Readline(c.log, c.br)
613 if err != nil && errors.Is(err, moxio.ErrLineTooLong) {
614 return "", fmt.Errorf("%s (%w)", err, errProtocol)
615 } else if err != nil {
616 return "", fmt.Errorf("%s (%w)", err, errIO)
621func (c *conn) lineChan() chan lineErr {
623 c.line = make(chan lineErr, 1)
625 line, err := c.readline0()
626 c.line <- lineErr{line, err}
632// readline from either the c.line channel, or otherwise read from connection.
633func (c *conn) xreadline(readCmd bool) string {
639 line, err = le.line, le.err
641 line, err = c.readline0()
644 if readCmd && errors.Is(err, os.ErrDeadlineExceeded) {
645 err := c.conn.SetDeadline(time.Now().Add(10 * time.Second))
646 c.log.Check(err, "setting deadline")
647 c.xwritelinef("* BYE inactive")
650 if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) {
651 c.xbrokenf("%s (%w)", err, errIO)
657 // We typically respond immediately (IDLE is an exception).
658 // The client may not be reading, or may have disappeared.
659 // Don't wait more than 5 minutes before closing down the connection.
660 // The write deadline is managed in IDLE as well.
661 // For unauthenticated connections, we require the client to read faster.
662 wd := 5 * time.Minute
663 if c.state == stateNotAuthenticated {
664 wd = 30 * time.Second
666 err = c.conn.SetWriteDeadline(time.Now().Add(wd))
667 c.log.Check(err, "setting write deadline")
672// write tagged command response, but first write pending changes.
673func (c *conn) xwriteresultf(format string, args ...any) {
674 c.xbwriteresultf(format, args...)
678// write buffered tagged command response, but first write pending changes.
679func (c *conn) xbwriteresultf(format string, args ...any) {
681 case "fetch", "store", "search":
683 case "select", "examine":
684 // We don't send changes before having confirmed opening the mailbox, to prevent
685 // clients from trying to interpret changes when it considers there isn't a
686 // selected mailbox yet.
689 overflow, changes := c.comm.Get()
690 c.xapplyChanges(overflow, changes, true)
693 c.xbwritelinef(format, args...)
696func (c *conn) xwritelinef(format string, args ...any) {
697 c.xbwritelinef(format, args...)
701// Buffer line for write.
702func (c *conn) xbwritelinef(format string, args ...any) {
704 fmt.Fprintf(c.xbw, format, args...)
707func (c *conn) xflush() {
708 // If the connection is already broken, we're not going to write more.
714 xcheckf(err, "flush") // Should never happen, the Write caused by the Flush should panic on i/o error.
716 // If compression is enabled, we need to flush its stream.
718 // Note: Flush writes a sync message if there is nothing to flush. Ideally we
719 // wouldn't send that, but we would have to keep track of whether data needs to be
721 err := c.xflateWriter.Flush()
722 xcheckf(err, "flush deflate")
724 // The flate writer writes to a bufio.Writer, we must also flush that.
725 err = c.xflateBW.Flush()
726 xcheckf(err, "flush deflate writer")
730func (c *conn) parseCommand(tag *string, line string) (cmd string, p *parser) {
731 p = newParser(line, c)
737 return cmd, newParser(p.remainder(), c)
740func (c *conn) xreadliteral(size int64, sync bool) []byte {
744 buf := make([]byte, size)
746 if err := c.conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil {
747 c.log.Errorx("setting read deadline", err)
750 _, err := io.ReadFull(c.br, buf)
752 c.xbrokenf("reading literal: %s (%w)", err, errIO)
758var cleanClose struct{} // Sentinel value for panic/recover indicating clean close of connection.
760// serve handles a single IMAP connection on nc.
762// If xtls is set, immediate TLS should be enabled on the connection, unless
763// viaHTTP is set, which indicates TLS is already active with the connection coming
764// from the webserver with IMAP chosen through ALPN. activated. If viaHTTP is set,
765// the TLS config ddid not enable client certificate authentication. If xtls is
766// false and tlsConfig is set, STARTTLS may enable TLS later on.
768// If noRequireSTARTTLS is set, TLS is not required for authentication.
770// If accountAddress is not empty, it is the email address of the account to open
773// The connection is closed before returning.
774func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, xtls, noTLSClientAuth, noRequireSTARTTLS, viaHTTPS bool, preauthAddress string) {
776 if a, ok := nc.RemoteAddr().(*net.TCPAddr); ok {
779 // For tests and for imapserve.
780 remoteIP = net.ParseIP("127.0.0.10")
788 noTLSClientAuth: noTLSClientAuth,
790 baseTLSConfig: tlsConfig,
792 noRequireSTARTTLS: noRequireSTARTTLS,
793 enabled: map[capability]bool{},
795 cmdStart: time.Now(),
797 var logmutex sync.Mutex
798 // Also see (and possibly update) c.logbg, for logging in a goroutine.
799 c.log = mlog.New("imapserver", nil).WithFunc(func() []slog.Attr {
801 defer logmutex.Unlock()
804 slog.Int64("cid", c.cid),
805 slog.Duration("delta", now.Sub(c.lastlog)),
808 if c.username != "" {
809 l = append(l, slog.String("username", c.username))
813 c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn)
814 // todo: tracing should be done on whatever comes out of c.br. the remote connection write a command plus data, and bufio can read it in one read, causing a command parser that sets the tracing level to data to have no effect. we are now typically logging sent messages, when mail clients append to the Sent mailbox.
815 c.br = bufio.NewReader(c.tr)
816 c.xtw = moxio.NewTraceWriter(c.log, "S: ", c)
817 c.xbw = bufio.NewWriter(c.xtw)
819 // Many IMAP connections use IDLE to wait for new incoming messages. We'll enable
820 // keepalive to get a higher chance of the connection staying alive, or otherwise
821 // detecting broken connections early.
824 tcpconn = nc.(*tls.Conn).NetConn()
826 if tc, ok := tcpconn.(*net.TCPConn); ok {
827 if err := tc.SetKeepAlivePeriod(5 * time.Minute); err != nil {
828 c.log.Errorx("setting keepalive period", err)
829 } else if err := tc.SetKeepAlive(true); err != nil {
830 c.log.Errorx("enabling keepalive", err)
834 c.log.Info("new connection",
835 slog.Any("remote", c.conn.RemoteAddr()),
836 slog.Any("local", c.conn.LocalAddr()),
837 slog.Bool("tls", xtls),
838 slog.Bool("viahttps", viaHTTPS),
839 slog.String("listener", listenerName))
842 err := c.conn.Close()
844 c.log.Debugx("closing connection", err)
847 // If changes for NOTIFY's SELECTED-DELAYED are still pending, we'll acknowledge
848 // their message removals so the files can be erased.
849 c.flushNotifyDelayed()
851 if c.account != nil {
853 err := c.account.Close()
854 c.xsanity(err, "close account")
860 if x == nil || x == cleanClose {
861 c.log.Info("connection closed")
862 } else if err, ok := x.(error); ok && isClosed(err) {
863 c.log.Infox("connection closed", err)
865 c.log.Error("unhandled panic", slog.Any("err", x))
867 metrics.PanicInc(metrics.Imapserver)
868 unhandledPanics.Add(1) // For tests.
872 if xtls && !viaHTTPS {
873 // Start TLS on connection. We perform the handshake explicitly, so we can set a
874 // timeout, do client certificate authentication, log TLS details afterwards.
875 c.xtlsHandshakeAndAuthenticate(c.conn)
879 case <-mox.Shutdown.Done():
881 c.xwritelinef("* BYE mox shutting down")
886 if !limiterConnectionrate.Add(c.remoteIP, time.Now(), 1) {
887 c.xwritelinef("* BYE connection rate from your ip or network too high, slow down please")
891 // If remote IP/network resulted in too many authentication failures, refuse to serve.
892 if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
893 metrics.AuthenticationRatelimitedInc("imap")
894 c.log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", c.remoteIP))
895 c.xwritelinef("* BYE too many auth failures")
899 if !limiterConnections.Add(c.remoteIP, time.Now(), 1) {
900 c.log.Debug("refusing connection due to many open connections", slog.Any("remoteip", c.remoteIP))
901 c.xwritelinef("* BYE too many open connections from your ip or network")
904 defer limiterConnections.Add(c.remoteIP, time.Now(), -1)
906 // We register and unregister the original connection, in case it c.conn is
907 // replaced with a TLS connection later on.
908 mox.Connections.Register(nc, "imap", listenerName)
909 defer mox.Connections.Unregister(nc)
911 if preauthAddress != "" {
912 acc, _, _, err := store.OpenEmail(c.log, preauthAddress, false)
914 c.log.Debugx("open account for preauth address", err, slog.String("address", preauthAddress))
915 c.xwritelinef("* BYE open account for address: %s", err)
918 c.username = preauthAddress
920 c.comm = store.RegisterComm(c.account)
923 if c.account != nil && !c.noPreauth {
924 c.state = stateAuthenticated
925 c.xwritelinef("* PREAUTH [CAPABILITY %s] mox imap welcomes %s", c.capabilities(), c.username)
927 c.xwritelinef("* OK [CAPABILITY %s] mox imap", c.capabilities())
930 // Ensure any pending loginAttempt is written before we stop.
932 if c.loginAttempt != nil {
933 store.LoginAttemptAdd(context.Background(), c.logbg(), *c.loginAttempt)
935 c.loginAttemptTime = time.Time{}
941 c.xflush() // For flushing errors, or commands that did not flush explicitly.
943 // Flush login attempt if it hasn't already been flushed by an ID command within 1s
944 // after authentication.
945 if c.loginAttempt != nil && (c.loginAttempt.UserAgent != "" || time.Since(c.loginAttemptTime) >= time.Second) {
946 store.LoginAttemptAdd(context.Background(), c.logbg(), *c.loginAttempt)
948 c.loginAttemptTime = time.Time{}
953// isClosed returns whether i/o failed, typically because the connection is closed.
954// For connection errors, we often want to generate fewer logs.
955func isClosed(err error) bool {
956 return errors.Is(err, errIO) || errors.Is(err, errProtocol) || mlog.IsClosed(err)
959// newLoginAttempt initializes a c.loginAttempt, for adding to the store after
960// filling in the results and other details.
961func (c *conn) newLoginAttempt(useTLS bool, authMech string) {
962 if c.loginAttempt != nil {
963 store.LoginAttemptAdd(context.Background(), c.logbg(), *c.loginAttempt)
966 c.loginAttemptTime = time.Now()
968 var state *tls.ConnectionState
969 if tc, ok := c.conn.(*tls.Conn); ok && useTLS {
970 v := tc.ConnectionState()
974 localAddr := c.conn.LocalAddr().String()
975 localIP, _, _ := net.SplitHostPort(localAddr)
980 c.loginAttempt = &store.LoginAttempt{
981 RemoteIP: c.remoteIP.String(),
983 TLS: store.LoginAttemptTLS(state),
985 UserAgent: c.userAgent, // May still be empty, to be filled in later.
987 Result: store.AuthError, // Replaced by caller.
991// makeTLSConfig makes a new tls config that is bound to the connection for
992// possible client certificate authentication.
993func (c *conn) makeTLSConfig() *tls.Config {
994 // We clone the config so we can set VerifyPeerCertificate below to a method bound
995 // to this connection. Earlier, we set session keys explicitly on the base TLS
996 // config, so they can be used for this connection too.
997 tlsConf := c.baseTLSConfig.Clone()
999 if c.noTLSClientAuth {
1003 // Allow client certificate authentication, for use with the sasl "external"
1004 // authentication mechanism.
1005 tlsConf.ClientAuth = tls.RequestClientCert
1007 // We verify the client certificate during the handshake. The TLS handshake is
1008 // initiated explicitly for incoming connections and during starttls, so we can
1009 // immediately extract the account name and address used for authentication.
1010 tlsConf.VerifyPeerCertificate = c.tlsClientAuthVerifyPeerCert
1015// tlsClientAuthVerifyPeerCert can be used as tls.Config.VerifyPeerCertificate, and
1016// sets authentication-related fields on conn. This is not called on resumed TLS
1018func (c *conn) tlsClientAuthVerifyPeerCert(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
1019 if len(rawCerts) == 0 {
1023 // If we had too many authentication failures from this IP, don't attempt
1024 // authentication. If this is a new incoming connetion, it is closed after the TLS
1026 if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
1030 cert, err := x509.ParseCertificate(rawCerts[0])
1032 c.log.Debugx("parsing tls client certificate", err)
1035 if err := c.tlsClientAuthVerifyPeerCertParsed(cert); err != nil {
1036 c.log.Debugx("verifying tls client certificate", err)
1037 return fmt.Errorf("verifying client certificate: %w", err)
1042// tlsClientAuthVerifyPeerCertParsed verifies a client certificate. Called both for
1043// fresh and resumed TLS connections.
1044func (c *conn) tlsClientAuthVerifyPeerCertParsed(cert *x509.Certificate) error {
1045 if c.account != nil {
1046 return fmt.Errorf("cannot authenticate with tls client certificate after previous authentication")
1049 // todo: it would be nice to postpone storing the loginattempt for tls pubkey auth until we have the ID command. but delaying is complicated because we can't get the tls information in this function. that's why we store the login attempt in a goroutine below, where it can can get a lock when accessing the tls connection only when this function has returned. we can't access c.loginAttempt (we would turn it into a slice) in a goroutine without adding more locking. for now we'll do without user-agent/id for tls pub key auth.
1050 c.newLoginAttempt(false, "tlsclientauth")
1052 // Get TLS connection state in goroutine because we are called while performing the
1053 // TLS handshake, which already has the tls connection locked.
1054 conn := c.conn.(*tls.Conn)
1055 la := *c.loginAttempt
1056 c.loginAttempt = nil
1057 logbg := c.logbg() // Evaluate attributes now, can't do it in goroutine.
1060 // In case of panic don't take the whole program down.
1063 c.log.Error("recover from panic", slog.Any("panic", x))
1065 metrics.PanicInc(metrics.Imapserver)
1069 state := conn.ConnectionState()
1070 la.TLS = store.LoginAttemptTLS(&state)
1071 store.LoginAttemptAdd(context.Background(), logbg, la)
1074 if la.Result == store.AuthSuccess {
1075 mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
1077 mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
1081 // For many failed auth attempts, slow down verification attempts.
1082 if c.authFailed > 3 && authFailDelay > 0 {
1083 mox.Sleep(mox.Context, time.Duration(c.authFailed-3)*authFailDelay)
1085 c.authFailed++ // Compensated on success.
1087 // On the 3rd failed authentication, start responding slowly. Successful auth will
1088 // cause fast responses again.
1089 if c.authFailed >= 3 {
1094 shabuf := sha256.Sum256(cert.RawSubjectPublicKeyInfo)
1095 fp := base64.RawURLEncoding.EncodeToString(shabuf[:])
1096 c.loginAttempt.TLSPubKeyFingerprint = fp
1097 pubKey, err := store.TLSPublicKeyGet(context.TODO(), fp)
1099 if err == bstore.ErrAbsent {
1100 c.loginAttempt.Result = store.AuthBadCredentials
1102 return fmt.Errorf("looking up tls public key with fingerprint %s, subject %q, issuer %q: %v", fp, cert.Subject, cert.Issuer, err)
1104 c.loginAttempt.LoginAddress = pubKey.LoginAddress
1106 // Verify account exists and still matches address. We don't check for account
1107 // login being disabled if preauth is disabled. In that case, sasl external auth
1108 // will be done before credentials can be used, and login disabled will be checked
1109 // then, where it will result in a more helpful error message.
1110 checkLoginDisabled := !pubKey.NoIMAPPreauth
1111 acc, accName, _, err := store.OpenEmail(c.log, pubKey.LoginAddress, checkLoginDisabled)
1112 c.loginAttempt.AccountName = accName
1114 if errors.Is(err, store.ErrLoginDisabled) {
1115 c.loginAttempt.Result = store.AuthLoginDisabled
1117 // note: we cannot send a more helpful error message to the client.
1118 return fmt.Errorf("opening account for address %s for public key %s: %w", pubKey.LoginAddress, fp, err)
1123 c.xsanity(err, "close account")
1126 c.loginAttempt.AccountName = acc.Name
1127 if acc.Name != pubKey.Account {
1128 return fmt.Errorf("tls client public key %s is for account %s, but email address %s is for account %s", fp, pubKey.Account, pubKey.LoginAddress, acc.Name)
1131 c.loginAttempt.Result = store.AuthSuccess
1134 c.noPreauth = pubKey.NoIMAPPreauth
1136 acc = nil // Prevent cleanup by defer.
1137 c.username = pubKey.LoginAddress
1138 c.comm = store.RegisterComm(c.account)
1139 c.log.Debug("tls client authenticated with client certificate",
1140 slog.String("fingerprint", fp),
1141 slog.String("username", c.username),
1142 slog.String("account", c.account.Name),
1143 slog.Any("remote", c.remoteIP))
1147// xtlsHandshakeAndAuthenticate performs the TLS handshake, and verifies a client
1148// certificate if present.
1149func (c *conn) xtlsHandshakeAndAuthenticate(conn net.Conn) {
1150 tlsConn := tls.Server(conn, c.makeTLSConfig())
1152 c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn)
1153 c.br = bufio.NewReader(c.tr)
1155 cidctx := context.WithValue(mox.Context, mlog.CidKey, c.cid)
1156 ctx, cancel := context.WithTimeout(cidctx, time.Minute)
1158 c.log.Debug("starting tls server handshake")
1159 if err := tlsConn.HandshakeContext(ctx); err != nil {
1160 c.xbrokenf("tls handshake: %s (%w)", err, errIO)
1164 cs := tlsConn.ConnectionState()
1165 if cs.DidResume && len(cs.PeerCertificates) > 0 && !c.noTLSClientAuth {
1166 // Verify client after session resumption.
1167 err := c.tlsClientAuthVerifyPeerCertParsed(cs.PeerCertificates[0])
1169 c.xwritelinef("* BYE [ALERT] Error verifying client certificate after TLS session resumption: %s", err)
1170 c.xbrokenf("tls verify client certificate after resumption: %s (%w)", err, errIO)
1174 version, ciphersuite := moxio.TLSInfo(cs)
1175 attrs := []slog.Attr{
1176 slog.String("version", version),
1177 slog.String("ciphersuite", ciphersuite),
1178 slog.String("sni", cs.ServerName),
1179 slog.Bool("resumed", cs.DidResume),
1180 slog.Bool("notlsclientauth", c.noTLSClientAuth),
1181 slog.Int("clientcerts", len(cs.PeerCertificates)),
1183 if c.account != nil {
1184 attrs = append(attrs,
1185 slog.String("account", c.account.Name),
1186 slog.String("username", c.username),
1189 c.log.Debug("tls handshake completed", attrs...)
1192func (c *conn) command() {
1193 var tag, cmd, cmdlow string
1199 metricIMAPCommands.WithLabelValues(c.cmdMetric, result).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second))
1202 logFields := []slog.Attr{
1203 slog.String("cmd", c.cmd),
1204 slog.Duration("duration", time.Since(c.cmdStart)),
1209 if x == nil || x == cleanClose {
1210 c.log.Debug("imap command done", logFields...)
1212 if x == cleanClose {
1213 // If compression was enabled, we flush & close the deflate stream.
1215 // Note: Close and flush can Write and may panic with an i/o error.
1216 if err := c.xflateWriter.Close(); err != nil {
1217 c.log.Debugx("close deflate writer", err)
1218 } else if err := c.xflateBW.Flush(); err != nil {
1219 c.log.Debugx("flush deflate buffer", err)
1227 err, ok := x.(error)
1229 c.log.Error("imap command panic", append([]slog.Attr{slog.Any("panic", x)}, logFields...)...)
1234 var sxerr syntaxError
1236 var serr serverError
1238 c.log.Infox("imap command ioerror", err, logFields...)
1240 if errors.Is(err, errProtocol) {
1244 } else if errors.As(err, &sxerr) {
1245 result = "badsyntax"
1247 // Other side is likely speaking something else than IMAP, send error message and
1248 // stop processing because there is a good chance whatever they sent has multiple
1250 c.xwritelinef("* BYE please try again speaking imap")
1251 c.xbrokenf("not speaking imap (%w)", errIO)
1253 c.log.Debugx("imap command syntax error", sxerr.err, logFields...)
1254 c.log.Info("imap syntax error", slog.String("lastline", c.lastLine))
1255 fatal := strings.HasSuffix(c.lastLine, "+}")
1257 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
1258 c.log.Check(err, "setting write deadline")
1260 if sxerr.line != "" {
1261 c.xbwritelinef("%s", sxerr.line)
1264 if sxerr.code != "" {
1265 code = "[" + sxerr.code + "] "
1267 c.xbwriteresultf("%s BAD %s%s unrecognized syntax/command: %v", tag, code, cmd, sxerr.errmsg)
1270 panic(fmt.Errorf("aborting connection after syntax error for command with non-sync literal: %w", errProtocol))
1272 } else if errors.As(err, &serr) {
1273 result = "servererror"
1274 c.log.Errorx("imap command server error", err, logFields...)
1276 c.xbwriteresultf("%s NO %s %v", tag, cmd, err)
1277 } else if errors.As(err, &uerr) {
1278 result = "usererror"
1279 c.log.Debugx("imap command user error", err, logFields...)
1280 if uerr.code != "" {
1281 c.xbwriteresultf("%s NO [%s] %s %v", tag, uerr.code, cmd, err)
1283 c.xbwriteresultf("%s NO %s %v", tag, cmd, err)
1286 // Other type of panic, we pass it on, aborting the connection.
1288 c.log.Errorx("imap command panic", err, logFields...)
1295 // If NOTIFY is enabled, we wait for either a line (with a command) from the
1296 // client, or a change event. If we see a line, we continue below as for the
1297 // non-NOTIFY case, parsing the command.
1299 if c.notify != nil {
1303 case le := <-c.lineChan():
1305 if err := le.err; err != nil {
1306 if errors.Is(err, os.ErrDeadlineExceeded) {
1307 err := c.conn.SetDeadline(time.Now().Add(10 * time.Second))
1308 c.log.Check(err, "setting write deadline")
1309 c.xwritelinef("* BYE inactive")
1312 if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) {
1313 c.xbrokenf("%s (%w)", err, errIO)
1320 case <-c.comm.Pending:
1321 overflow, changes := c.comm.Get()
1322 c.xapplyChanges(overflow, changes, false)
1325 case <-mox.Shutdown.Done():
1327 c.xwritelinef("* BYE shutting down")
1328 c.xbrokenf("shutting down (%w)", errIO)
1332 // Reset the write deadline. In case of little activity, with a command timeout of
1333 // 30 minutes, we have likely passed it.
1334 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
1335 c.log.Check(err, "setting write deadline")
1337 // Without NOTIFY, we just read a line.
1338 line = c.xreadline(true)
1340 cmd, p = c.parseCommand(&tag, line)
1341 cmdlow = strings.ToLower(cmd)
1343 c.cmdStart = time.Now()
1344 c.cmdMetric = "(unrecognized)"
1347 case <-mox.Shutdown.Done():
1349 c.xwritelinef("* BYE shutting down")
1350 c.xbrokenf("shutting down (%w)", errIO)
1354 fn := commands[cmdlow]
1356 xsyntaxErrorf("unknown command %q", cmd)
1361 // Check if command is allowed in this state.
1362 if _, ok1 := commandsStateAny[cmdlow]; ok1 {
1363 } else if _, ok2 := commandsStateNotAuthenticated[cmdlow]; ok2 && c.state == stateNotAuthenticated {
1364 } else if _, ok3 := commandsStateAuthenticated[cmdlow]; ok3 && (c.state == stateAuthenticated || c.state == stateSelected) {
1365 } else if _, ok4 := commandsStateSelected[cmdlow]; ok4 && c.state == stateSelected {
1366 } else if ok1 || ok2 || ok3 || ok4 {
1367 xuserErrorf("not allowed in this connection state")
1369 xserverErrorf("unrecognized command")
1373 if _, ok := commandsSequence[cmdlow]; ok && c.uidonly {
1374 xsyntaxCodeErrorf("UIDREQUIRED", "cannot use message sequence numbers with uidonly")
1380func (c *conn) broadcast(changes []store.Change) {
1381 if len(changes) == 0 {
1384 c.log.Debug("broadcast changes", slog.Any("changes", changes))
1385 c.comm.Broadcast(changes)
1388// matchStringer matches a string against reference + mailbox patterns.
1389type matchStringer interface {
1390 MatchString(s string) bool
1393type noMatch struct{}
1395// MatchString for noMatch always returns false.
1396func (noMatch) MatchString(s string) bool {
1400// xmailboxPatternMatcher returns a matcher for mailbox names given the reference and patterns.
1401// Patterns can include "%" and "*", matching any character excluding and including a slash respectively.
1402func xmailboxPatternMatcher(ref string, patterns []string) matchStringer {
1403 if strings.HasPrefix(ref, "/") {
1408 for _, pat := range patterns {
1409 if strings.HasPrefix(pat, "/") {
1415 s = path.Join(ref, pat)
1418 // Fix casing for all Inbox paths.
1419 first, _, _ := strings.Cut(s, "/")
1420 if strings.EqualFold(first, "Inbox") {
1421 s = "Inbox" + s[len("Inbox"):]
1425 var rs strings.Builder
1426 for _, c := range s {
1428 rs.WriteString("[^/]*")
1429 } else if c == '*' {
1430 rs.WriteString(".*")
1432 rs.WriteString(regexp.QuoteMeta(string(c)))
1435 subs = append(subs, rs.String())
1441 rs := "^(" + strings.Join(subs, "|") + ")$"
1442 re, err := regexp.Compile(rs)
1443 xcheckf(err, "compiling regexp for mailbox patterns")
1447func (c *conn) sequence(uid store.UID) msgseq {
1449 panic("sequence with uidonly")
1451 return uidSearch(c.uids, uid)
1454func uidSearch(uids []store.UID, uid store.UID) msgseq {
1461 return msgseq(i + 1)
1471func (c *conn) xsequence(uid store.UID) msgseq {
1473 panic("xsequence with uidonly")
1475 seq := c.sequence(uid)
1477 xserverErrorf("unknown uid %d (%w)", uid, errProtocol)
1482func (c *conn) sequenceRemove(seq msgseq, uid store.UID) {
1484 panic("sequenceRemove with uidonly")
1487 if c.uids[i] != uid {
1488 xserverErrorf("got uid %d at msgseq %d, expected uid %d", uid, seq, c.uids[i])
1490 copy(c.uids[i:], c.uids[i+1:])
1491 c.uids = c.uids[:c.exists-1]
1493 c.checkUIDs(c.uids, true)
1496// add uid to session, through c.uidnext, and if uidonly isn't enabled to c.uids.
1497// care must be taken that pending changes are fetched while holding the account
1498// wlock, and applied before adding this uid, because those pending changes may
1499// contain another new uid that has to be added first.
1500func (c *conn) uidAppend(uid store.UID) {
1502 if uid < c.uidnext {
1503 panic(fmt.Sprintf("new uid %d < uidnext %d", uid, c.uidnext))
1510 if uidSearch(c.uids, uid) > 0 {
1511 xserverErrorf("uid already present (%w)", errProtocol)
1513 if c.exists > 0 && uid < c.uids[c.exists-1] {
1514 xserverErrorf("new uid %d is smaller than last uid %d (%w)", uid, c.uids[c.exists-1], errProtocol)
1518 c.uids = append(c.uids, uid)
1519 c.checkUIDs(c.uids, true)
1522// sanity check that uids are in ascending order.
1523func (c *conn) checkUIDs(uids []store.UID, checkExists bool) {
1528 if checkExists && uint32(len(uids)) != c.exists {
1529 panic(fmt.Sprintf("exists %d does not match len(uids) %d", c.exists, len(c.uids)))
1532 for i, uid := range uids {
1533 if uid == 0 || i > 0 && uid <= uids[i-1] {
1534 xserverErrorf("bad uids %v", uids)
1539func slicesAny[T any](l []T) []any {
1540 r := make([]any, len(l))
1541 for i, v := range l {
1547// newCachedLastUID returns a method that returns the highest uid for a mailbox,
1548// for interpretation of "*". If mailboxID is for the selected mailbox, the UIDs
1549// visible in the session are taken into account. If there is no UID, 0 is
1550// returned. If an error occurs, xerrfn is called, which should not return.
1551func (c *conn) newCachedLastUID(tx *bstore.Tx, mailboxID int64, xerrfn func(err error)) func() store.UID {
1554 return func() store.UID {
1558 if c.mailboxID == mailboxID {
1563 return c.uids[c.exists-1]
1566 q := bstore.QueryTx[store.Message](tx)
1567 q.FilterNonzero(store.Message{MailboxID: mailboxID})
1568 q.FilterEqual("Expunged", false)
1569 if c.mailboxID == mailboxID {
1570 q.FilterLess("UID", c.uidnext)
1575 if err == bstore.ErrAbsent {
1581 panic(err) // xerrfn should have called panic.
1589// xnumSetEval evaluates nums to uids given the current session state and messages
1590// in the selected mailbox. The returned UIDs are sorted, without duplicates.
1591func (c *conn) xnumSetEval(tx *bstore.Tx, isUID bool, nums numSet) []store.UID {
1592 if nums.searchResult {
1593 // UIDs that do not exist can be ignored.
1598 // Update previously stored UIDs. Some may have been deleted.
1599 // Once deleted a UID will never come back, so we'll just remove those uids.
1601 var uids []store.UID
1602 if len(c.searchResult) > 0 {
1603 q := bstore.QueryTx[store.Message](tx)
1604 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
1605 q.FilterEqual("Expunged", false)
1606 q.FilterEqual("UID", slicesAny(c.searchResult)...)
1608 for m, err := range q.All() {
1609 xcheckf(err, "looking up messages from search result")
1610 uids = append(uids, m.UID)
1613 c.searchResult = uids
1616 for _, uid := range c.searchResult {
1617 if uidSearch(c.uids, uid) > 0 {
1618 c.searchResult[o] = uid
1622 c.searchResult = c.searchResult[:o]
1624 return c.searchResult
1628 uids := map[store.UID]struct{}{}
1630 // Sequence numbers that don't exist, or * on an empty mailbox, should result in a BAD response.
../rfc/9051:7018
1631 for _, r := range nums.ranges {
1635 xsyntaxErrorf("invalid seqset * on empty mailbox")
1637 ia = int(c.exists) - 1
1639 ia = int(r.first.number - 1)
1640 if ia >= int(c.exists) {
1641 xsyntaxErrorf("msgseq %d not in mailbox", r.first.number)
1645 uids[c.uids[ia]] = struct{}{}
1650 ib = int(c.exists) - 1
1652 ib = int(r.last.number - 1)
1653 if ib >= int(c.exists) {
1654 xsyntaxErrorf("msgseq %d not in mailbox", r.last.number)
1660 for _, uid := range c.uids[ia : ib+1] {
1661 uids[uid] = struct{}{}
1664 return slices.Sorted(maps.Keys(uids))
1667 // UIDs that do not exist can be ignored.
1672 uids := map[store.UID]struct{}{}
1675 xlastUID := c.newCachedLastUID(tx, c.mailboxID, func(xerr error) { xuserErrorf("%s", xerr) })
1676 for _, r := range nums.xinterpretStar(xlastUID).ranges {
1677 q := bstore.QueryTx[store.Message](tx)
1678 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
1679 q.FilterEqual("Expunged", false)
1681 q.FilterEqual("UID", r.first.number)
1683 q.FilterGreaterEqual("UID", r.first.number)
1684 q.FilterLessEqual("UID", r.last.number)
1686 q.FilterLess("UID", c.uidnext)
1688 for m, err := range q.All() {
1689 xcheckf(err, "enumerating uids")
1690 uids[m.UID] = struct{}{}
1693 return slices.Sorted(maps.Keys(uids))
1696 for _, r := range nums.ranges {
1702 uida := store.UID(r.first.number)
1704 uida = c.uids[c.exists-1]
1707 uidb := store.UID(last.number)
1709 uidb = c.uids[c.exists-1]
1713 uida, uidb = uidb, uida
1716 // Binary search for uida.
1721 if uida < c.uids[m] {
1723 } else if uida > c.uids[m] {
1730 for _, uid := range c.uids[s:] {
1731 if uid >= uida && uid <= uidb {
1732 uids[uid] = struct{}{}
1733 } else if uid > uidb {
1738 return slices.Sorted(maps.Keys(uids))
1741func (c *conn) ok(tag, cmd string) {
1742 c.xbwriteresultf("%s OK %s done", tag, cmd)
1746// xcheckmailboxname checks if name is valid, returning an INBOX-normalized name.
1747// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.
1748// Name is invalid if it contains leading/trailing/double slashes, or when it isn't
1749// unicode-normalized, or when empty or has special characters.
1750func xcheckmailboxname(name string, allowInbox bool) string {
1751 name, isinbox, err := config.CheckMailboxName(name, allowInbox)
1753 xuserErrorf("special mailboxname Inbox not allowed")
1754 } else if err != nil {
1755 xusercodeErrorf("CANNOT", "%s", err)
1760// Lookup mailbox by name.
1761// If the mailbox does not exist, panic is called with a user error.
1762// Must be called with account rlock held.
1763func (c *conn) xmailbox(tx *bstore.Tx, name string, missingErrCode string) store.Mailbox {
1764 mb, err := c.account.MailboxFind(tx, name)
1765 xcheckf(err, "finding mailbox")
1767 // missingErrCode can be empty, or e.g. TRYCREATE or ALREADYEXISTS.
1768 xusercodeErrorf(missingErrCode, "%w", store.ErrUnknownMailbox)
1773// Lookup mailbox by ID.
1774// If the mailbox does not exist, panic is called with a user error.
1775// Must be called with account rlock held.
1776func (c *conn) xmailboxID(tx *bstore.Tx, id int64) store.Mailbox {
1777 mb, err := store.MailboxID(tx, id)
1778 if err == bstore.ErrAbsent {
1779 xuserErrorf("%w", store.ErrUnknownMailbox)
1780 } else if err == store.ErrMailboxExpunged {
1782 xusercodeErrorf("NONEXISTENT", "mailbox has been deleted")
1787// Apply changes to our session state.
1788// Should not be called while holding locks, as changes are written to client connections, which can block.
1789// Does not flush output.
1790func (c *conn) xapplyChanges(overflow bool, changes []store.Change, sendDelayed bool) {
1791 // If more changes were generated than we can process, we send a
1794 if c.notify != nil && len(c.notify.Delayed) > 0 {
1795 changes = append(c.notify.Delayed, changes...)
1797 c.flushChanges(changes)
1798 // We must not send any more unsolicited untagged responses to the client for
1800 c.notify = ¬ify{}
1801 c.xbwritelinef("* OK [NOTIFICATIONOVERFLOW] out of sync after too many pending changes")
1805 // applyChanges for IDLE and NOTIFY. When explicitly in IDLE while NOTIFY is
1807 if c.notify != nil {
1808 c.xapplyChangesNotify(changes, sendDelayed)
1811 if len(changes) == 0 {
1815 // Even in the case of a panic (e.g. i/o errors), we must mark removals as seen.
1816 origChanges := changes
1818 for _, change := range origChanges {
1819 if ch, ok := change.(store.ChangeRemoveUIDs); ok {
1820 c.comm.RemovalSeen(ch)
1825 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
1826 c.log.Check(err, "setting write deadline")
1828 c.log.Debug("applying changes", slog.Any("changes", changes))
1830 // Only keep changes for the selected mailbox, and changes that are always relevant.
1831 var n []store.Change
1832 for _, change := range changes {
1834 switch ch := change.(type) {
1835 case store.ChangeAddUID:
1837 case store.ChangeRemoveUIDs:
1839 case store.ChangeFlags:
1841 case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription, store.ChangeRemoveSubscription:
1842 n = append(n, change)
1844 case store.ChangeAnnotation:
1845 // note: annotations may have a mailbox associated with them, but we pass all
1848 if c.enabled[capMetadata] {
1849 n = append(n, change)
1852 case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords, store.ChangeThread:
1854 panic(fmt.Errorf("missing case for %#v", change))
1856 if c.state == stateSelected && mbID == c.mailboxID {
1857 n = append(n, change)
1862 qresync := c.enabled[capQresync]
1863 condstore := c.enabled[capCondstore]
1866 for i < len(changes) {
1867 // First process all new uids. So we only send a single EXISTS.
1868 var adds []store.ChangeAddUID
1869 for ; i < len(changes); i++ {
1870 ch, ok := changes[i].(store.ChangeAddUID)
1875 adds = append(adds, ch)
1878 // Write the exists, and the UID and flags as well. Hopefully the client waits for
1879 // long enough after the EXISTS to see these messages, and doesn't request them
1880 // again with a FETCH.
1881 c.xbwritelinef("* %d EXISTS", c.exists)
1882 for _, add := range adds {
1883 var modseqStr string
1885 modseqStr = fmt.Sprintf(" MODSEQ (%d)", add.ModSeq.Client())
1889 c.xbwritelinef("* %d UIDFETCH (FLAGS %s%s)", add.UID, flaglist(add.Flags, add.Keywords).pack(c), modseqStr)
1891 seq := c.xsequence(add.UID)
1892 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, add.UID, flaglist(add.Flags, add.Keywords).pack(c), modseqStr)
1898 change := changes[i]
1901 switch ch := change.(type) {
1902 case store.ChangeRemoveUIDs:
1903 var vanishedUIDs numSet
1904 for _, uid := range ch.UIDs {
1908 vanishedUIDs.append(uint32(uid))
1912 seq := c.xsequence(uid)
1913 c.sequenceRemove(seq, uid)
1915 vanishedUIDs.append(uint32(uid))
1917 c.xbwritelinef("* %d EXPUNGE", seq)
1920 if !vanishedUIDs.empty() {
1922 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
1923 c.xbwritelinef("* VANISHED %s", s)
1927 case store.ChangeFlags:
1928 var modseqStr string
1930 modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client())
1934 c.xbwritelinef("* %d UIDFETCH (FLAGS %s%s)", ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
1936 // The uid can be unknown if we just expunged it while another session marked it as deleted just before.
1937 seq := c.sequence(ch.UID)
1941 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
1944 case store.ChangeRemoveMailbox:
1945 // Only announce \NonExistent to modern clients, otherwise they may ignore the
1946 // unrecognized \NonExistent and interpret this as a newly created mailbox, while
1947 // the goal was to remove it...
1948 if c.enabled[capIMAP4rev2] {
1949 c.xbwritelinef(`* LIST (\NonExistent) "/" %s`, mailboxt(ch.Name).pack(c))
1952 case store.ChangeAddMailbox:
1953 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), mailboxt(ch.Mailbox.Name).pack(c))
1955 case store.ChangeRenameMailbox:
1958 if c.enabled[capIMAP4rev2] {
1959 oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(ch.OldName).pack(c))
1961 c.xbwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), mailboxt(ch.NewName).pack(c), oldname)
1963 case store.ChangeAddSubscription:
1964 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.ListFlags...), " "), mailboxt(ch.MailboxName).pack(c))
1966 case store.ChangeRemoveSubscription:
1967 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.ListFlags, " "), mailboxt(ch.MailboxName).pack(c))
1969 case store.ChangeAnnotation:
1971 c.xbwritelinef(`* METADATA %s %s`, mailboxt(ch.MailboxName).pack(c), astring(ch.Key).pack(c))
1974 panic(fmt.Sprintf("internal error, missing case for %#v", change))
1979// xapplyChangesNotify is like xapplyChanges, but for NOTIFY, with configurable
1980// mailboxes to notify about, and configurable events to send, including which
1981// fetch attributes to return. All calls must go through xapplyChanges, for overflow
1983func (c *conn) xapplyChangesNotify(changes []store.Change, sendDelayed bool) {
1984 if sendDelayed && len(c.notify.Delayed) > 0 {
1985 changes = append(c.notify.Delayed, changes...)
1986 c.notify.Delayed = nil
1989 if len(changes) == 0 {
1993 // Even in the case of a panic (e.g. i/o errors), we must mark removals as seen.
1994 // For selected-delayed, we may have postponed handling the message, so we call
1995 // RemovalSeen when handling a change, and mark how far we got, so we only process
1996 // changes that we haven't processed yet.
1997 unhandled := changes
1999 for _, change := range unhandled {
2000 if ch, ok := change.(store.ChangeRemoveUIDs); ok {
2001 c.comm.RemovalSeen(ch)
2006 c.log.Debug("applying notify changes", slog.Any("changes", changes))
2008 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
2009 c.log.Check(err, "setting write deadline")
2011 qresync := c.enabled[capQresync]
2012 condstore := c.enabled[capCondstore]
2014 // Prepare for providing a read-only transaction on first-use, for MessageNew fetch
2019 err := tx.Rollback()
2020 c.log.Check(err, "rolling back tx")
2023 xtx := func() *bstore.Tx {
2029 tx, err = c.account.DB.Begin(context.TODO(), false)
2034 // On-demand mailbox lookups, with cache.
2035 mailboxes := map[int64]store.Mailbox{}
2036 xmailbox := func(id int64) store.Mailbox {
2037 if mb, ok := mailboxes[id]; ok {
2040 mb := store.Mailbox{ID: id}
2041 err := xtx().Get(&mb)
2042 xcheckf(err, "get mailbox")
2047 // Keep track of last command, to close any open message file (for fetching
2048 // attributes) in case of a panic.
2057 for index, change := range changes {
2058 switch ch := change.(type) {
2059 case store.ChangeAddUID:
2061 // todo:
../rfc/5465:525 group ChangeAddUID for the same mailbox, so we can send a single EXISTS. useful for imports.
2063 mb := xmailbox(ch.MailboxID)
2064 ms, ev, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMessageNew)
2069 // For non-selected mailbox, send STATUS with UIDNEXT, MESSAGES. And HIGESTMODSEQ
2071 // There is no mention of UNSEEN for MessageNew, but clients will want to show a
2072 // new "unread messages" count, and they will have to understand it since
2073 // FlagChange is specified as sending UNSEEN.
2074 if mb.ID != c.mailboxID {
2075 if condstore || qresync {
2076 c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d HIGHESTMODSEQ %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UID+1, ch.MessageCountIMAP, ch.ModSeq, ch.Unseen)
2078 c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UID+1, ch.MessageCountIMAP, ch.Unseen)
2083 // Delay sending all message events, we want to prevent synchronization issues
2085 if ms.Kind == mbspecSelectedDelayed && !sendDelayed {
2086 c.notify.Delayed = append(c.notify.Delayed, change)
2093 c.xbwritelinef("* %d EXISTS", c.exists)
2095 // If client did not specify attributes, we'll send the defaults.
2096 if len(ev.FetchAtt) == 0 {
2097 var modseqStr string
2099 modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client())
2101 // NOTIFY does not specify the default fetch attributes to return, we send UID and
2105 c.xbwritelinef("* %d UIDFETCH (FLAGS %s%s)", ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
2107 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", c.xsequence(ch.UID), ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
2112 // todo:
../rfc/5465:543 mark messages as \seen after processing if client didn't use the .PEEK-variants.
2113 cmd = &fetchCmd{conn: c, isUID: true, rtx: xtx(), mailboxID: ch.MailboxID, uid: ch.UID}
2114 data, err := cmd.process(ev.FetchAtt)
2116 // There is no good way to notify the client about errors. We continue below to
2117 // send a FETCH with just the UID. And we send an untagged NO in the hope a client
2118 // developer sees the message.
2119 c.log.Errorx("generating notify fetch response", err, slog.Int64("mailboxid", ch.MailboxID), slog.Any("uid", ch.UID))
2120 c.xbwritelinef("* NO generating notify fetch response: %s", err.Error())
2121 // Always add UID, also for uidonly, to ensure a non-empty list.
2122 data = listspace{bare("UID"), number(ch.UID)}
2126 fmt.Fprintf(cmd.conn.xbw, "* %d UIDFETCH ", ch.UID)
2128 fmt.Fprintf(cmd.conn.xbw, "* %d FETCH ", c.xsequence(ch.UID))
2131 defer c.xtracewrite(mlog.LevelTracedata)()
2132 data.xwriteTo(cmd.conn, cmd.conn.xbw)
2133 c.xtracewrite(mlog.LevelTrace) // Restore.
2134 cmd.conn.xbw.Write([]byte("\r\n"))
2140 case store.ChangeRemoveUIDs:
2142 mb := xmailbox(ch.MailboxID)
2143 ms, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMessageExpunge)
2145 unhandled = changes[index+1:]
2146 c.comm.RemovalSeen(ch)
2150 // For non-selected mailboxes, we send STATUS with at least UIDNEXT and MESSAGES.
2152 // In case of QRESYNC, we send HIGHESTMODSEQ. Also for CONDSTORE, which isn't
2155 // There is no mention of UNSEEN, but clients will want to show a new "unread
2156 // messages" count, and they can parse it since FlagChange is specified as sending
2158 if mb.ID != c.mailboxID {
2159 unhandled = changes[index+1:]
2160 c.comm.RemovalSeen(ch)
2161 if condstore || qresync {
2162 c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d HIGHESTMODSEQ %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UIDNext, ch.MessageCountIMAP, ch.ModSeq, ch.Unseen)
2164 c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UIDNext, ch.MessageCountIMAP, ch.Unseen)
2169 // Delay sending all message events, we want to prevent synchronization issues
2171 if ms.Kind == mbspecSelectedDelayed && !sendDelayed {
2172 unhandled = changes[index+1:] // We'll call RemovalSeen in the future.
2173 c.notify.Delayed = append(c.notify.Delayed, change)
2177 unhandled = changes[index+1:]
2178 c.comm.RemovalSeen(ch)
2180 var vanishedUIDs numSet
2181 for _, uid := range ch.UIDs {
2185 vanishedUIDs.append(uint32(uid))
2189 seq := c.xsequence(uid)
2190 c.sequenceRemove(seq, uid)
2192 vanishedUIDs.append(uint32(uid))
2194 c.xbwritelinef("* %d EXPUNGE", seq)
2197 if !vanishedUIDs.empty() {
2199 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
2200 c.xbwritelinef("* VANISHED %s", s)
2204 case store.ChangeFlags:
2206 mb := xmailbox(ch.MailboxID)
2207 ms, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventFlagChange)
2210 } else if mb.ID != c.mailboxID {
2213 // We include UNSEEN, so clients can update the number of unread messages.
../rfc/5465:479
2214 if condstore || qresync {
2215 c.xbwritelinef("* STATUS %s (HIGHESTMODSEQ %d UIDVALIDITY %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.ModSeq, ch.UIDValidity, ch.Unseen)
2217 c.xbwritelinef("* STATUS %s (UIDVALIDITY %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UIDValidity, ch.Unseen)
2222 // Delay sending all message events, we want to prevent synchronization issues
2224 if ms.Kind == mbspecSelectedDelayed && !sendDelayed {
2225 c.notify.Delayed = append(c.notify.Delayed, change)
2229 // The uid can be unknown if we just expunged it while another session marked it as deleted just before.
2232 seq = c.sequence(ch.UID)
2238 var modseqStr string
2240 modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client())
2245 c.xbwritelinef("* %d UIDFETCH (FLAGS %s%s)", ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
2247 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
2250 case store.ChangeThread:
2254 case store.ChangeRemoveMailbox:
2255 mb := xmailbox(ch.MailboxID)
2256 _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxName)
2262 c.xbwritelinef(`* LIST (\NonExistent) "/" %s`, mailboxt(ch.Name).pack(c))
2264 case store.ChangeAddMailbox:
2265 mb := xmailbox(ch.Mailbox.ID)
2266 _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxName)
2270 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), mailboxt(ch.Mailbox.Name).pack(c))
2272 case store.ChangeRenameMailbox:
2273 mb := xmailbox(ch.MailboxID)
2274 _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxName)
2279 oldname := fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(ch.OldName).pack(c))
2280 c.xbwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), mailboxt(ch.NewName).pack(c), oldname)
2283 case store.ChangeAddSubscription:
2284 _, _, ok := c.notify.match(c, xtx, 0, ch.MailboxName, eventSubscriptionChange)
2288 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.ListFlags...), " "), mailboxt(ch.MailboxName).pack(c))
2290 case store.ChangeRemoveSubscription:
2291 _, _, ok := c.notify.match(c, xtx, 0, ch.MailboxName, eventSubscriptionChange)
2296 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.ListFlags, " "), mailboxt(ch.MailboxName).pack(c))
2298 case store.ChangeMailboxCounts:
2301 case store.ChangeMailboxSpecialUse:
2302 // todo: can we send special-use flags as part of an untagged LIST response?
2305 case store.ChangeMailboxKeywords:
2307 mb := xmailbox(ch.MailboxID)
2308 ms, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventFlagChange)
2311 } else if mb.ID != c.mailboxID {
2315 // Delay sending all message events, we want to prevent synchronization issues
2317 // This change is about mailbox keywords, but it's specified under the FlagChange
2320 if ms.Kind == mbspecSelectedDelayed && !sendDelayed {
2321 c.notify.Delayed = append(c.notify.Delayed, change)
2326 if len(ch.Keywords) > 0 {
2327 keywords = " " + strings.Join(ch.Keywords, " ")
2329 c.xbwritelinef(`* FLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent%s)`, keywords)
2331 case store.ChangeAnnotation:
2332 // Client does not have to enable METADATA/METADATA-SERVER. Just asking for these
2333 // events is enough.
2336 if ch.MailboxID == 0 {
2338 _, _, ok := c.notify.match(c, xtx, 0, "", eventServerMetadataChange)
2344 mb := xmailbox(ch.MailboxID)
2345 _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxMetadataChange)
2354 c.xbwritelinef(`* METADATA %s %s`, mailboxt(ch.MailboxName).pack(c), astring(ch.Key).pack(c))
2357 panic(fmt.Sprintf("internal error, missing case for %#v", change))
2361 // If we have too many delayed changes, we will warn about notification overflow,
2363 if len(c.notify.Delayed) > selectedDelayedChangesMax {
2364 l := c.notify.Delayed
2365 c.notify.Delayed = nil
2368 c.notify = ¬ify{}
2369 c.xbwritelinef("* OK [NOTIFICATIONOVERFLOW] out of sync after too many pending changes for selected mailbox")
2373// Capability returns the capabilities this server implements and currently has
2374// available given the connection state.
2377func (c *conn) cmdCapability(tag, cmd string, p *parser) {
2383 caps := c.capabilities()
2386 c.xbwritelinef("* CAPABILITY %s", caps)
2390// capabilities returns non-empty string with available capabilities based on connection state.
2391// For use in cmdCapability and untagged OK responses on connection start, login and authenticate.
2392func (c *conn) capabilities() string {
2393 caps := serverCapabilities
2394 if c.account != nil {
2395 conf, _ := c.account.Conf()
2396 if len(conf.IMAPCapabilitiesDisabled) > 0 {
2397 l := make([]string, 0, len(serverCapabilitiesList))
2398 for _, cap := range serverCapabilitiesList {
2399 if !slices.Contains(conf.IMAPCapabilitiesDisabled, strings.ToUpper(cap)) {
2403 caps = strings.Join(l, " ")
2408 // We only allow starting without TLS when explicitly configured, in violation of RFC.
2409 if !c.tls && c.baseTLSConfig != nil {
2412 if c.tls || c.noRequireSTARTTLS {
2413 caps += " AUTH=PLAIN"
2415 caps += " LOGINDISABLED"
2417 if c.tls && len(c.conn.(*tls.Conn).ConnectionState().PeerCertificates) > 0 && !c.viaHTTPS && !c.noTLSClientAuth {
2418 caps += " AUTH=EXTERNAL"
2423// No op, but useful for retrieving pending changes as untagged responses, e.g. of
2427func (c *conn) cmdNoop(tag, cmd string, p *parser) {
2435// Logout, after which server closes the connection.
2438func (c *conn) cmdLogout(tag, cmd string, p *parser) {
2445 c.state = stateNotAuthenticated
2447 c.xbwritelinef("* BYE thanks")
2452// Clients can use ID to tell the server which software they are using. Servers can
2453// respond with their version. For statistics/logging/debugging purposes.
2456func (c *conn) cmdID(tag, cmd string, p *parser) {
2461 var params map[string]string
2464 params = map[string]string{}
2466 if len(params) > 0 {
2472 if _, ok := params[k]; ok {
2473 xsyntaxErrorf("duplicate key %q", k)
2476 values = append(values, fmt.Sprintf("%s=%q", k, v))
2483 c.userAgent = strings.Join(values, " ")
2485 // The ID command is typically sent soon after authentication. So we've prepared
2486 // the LoginAttempt and write it now.
2487 if c.loginAttempt != nil {
2488 c.loginAttempt.UserAgent = c.userAgent
2489 store.LoginAttemptAdd(context.Background(), c.logbg(), *c.loginAttempt)
2490 c.loginAttempt = nil
2491 c.loginAttemptTime = time.Time{}
2494 // We just log the client id.
2495 c.log.Info("client id", slog.Any("params", params))
2499 if c.state == stateAuthenticated || c.state == stateSelected {
2500 c.xbwritelinef(`* ID ("name" "mox" "version" %s)`, string0(moxvar.Version).pack(c))
2502 c.xbwritelinef(`* ID ("name" "mox")`)
2507// Compress enables compression on the connection. Deflate is the only algorithm
2508// specified. TLS doesn't do compression nowadays, so we don't have to check for that.
2510// Status: Authenticated. The RFC doesn't mention this in prose, but the command is
2511// added to ABNF production rule "command-auth".
2512func (c *conn) cmdCompress(tag, cmd string, p *parser) {
2520 // Will do compression only once.
2523 xusercodeErrorf("COMPRESSIONACTIVE", "compression already active with previous compress command")
2526 if !strings.EqualFold(alg, "deflate") {
2527 xuserErrorf("compression algorithm not supported")
2530 // We must flush now, before we initialize flate.
2531 c.log.Debug("compression enabled")
2534 c.xflateBW = bufio.NewWriter(c)
2535 fw0, err := flate.NewWriter(c.xflateBW, flate.DefaultCompression)
2536 xcheckf(err, "deflate") // Cannot happen.
2537 xfw := moxio.NewFlateWriter(fw0)
2540 c.xflateWriter = xfw
2541 c.xtw = moxio.NewTraceWriter(c.log, "S: ", c.xflateWriter)
2542 c.xbw = bufio.NewWriter(c.xtw) // The previous c.xbw will not have buffered data.
2544 rc := xprefixConn(c.conn, c.br) // c.br may contain buffered data.
2545 // We use the special partial reader. Some clients write commands and flush the
2546 // buffer in "partial flush" mode instead of "sync flush" mode. The "sync flush"
2547 // mode emits an explicit zero-length data block that triggers the Go stdlib flate
2548 // reader to return data to us. It wouldn't for blocks written in "partial flush"
2549 // mode, and it would block us indefinitely while trying to read another flate
2550 // block. The partial reader returns data earlier, but still eagerly consumes all
2551 // blocks in its buffer.
2552 // todo: also _write_ in partial mode since it uses fewer bytes than a sync flush (which needs an additional 4 bytes for the zero-length data block). we need a writer that can flush in partial mode first. writing with sync flush will work with clients that themselves write with partial flush.
2553 fr := flate.NewReaderPartial(rc)
2554 c.tr = moxio.NewTraceReader(c.log, "C: ", fr)
2555 c.br = bufio.NewReader(c.tr)
2558// STARTTLS enables TLS on the connection, after a plain text start.
2559// Only allowed if TLS isn't already enabled, either through connecting to a
2560// TLS-enabled TCP port, or a previous STARTTLS command.
2561// After STARTTLS, plain text authentication typically becomes available.
2563// Status: Not authenticated.
2564func (c *conn) cmdStarttls(tag, cmd string, p *parser) {
2573 if c.baseTLSConfig == nil {
2574 xsyntaxErrorf("starttls not announced")
2577 conn := xprefixConn(c.conn, c.br)
2578 // We add the cid to facilitate debugging in case of TLS connection failure.
2579 c.ok(tag, cmd+" ("+mox.ReceivedID(c.cid)+")")
2581 c.xtlsHandshakeAndAuthenticate(conn)
2584 // We are not sending unsolicited CAPABILITIES for newly available authentication
2585 // mechanisms, clients can't depend on us sending it and should ask it themselves.
2589// Authenticate using SASL. Supports multiple back and forths between client and
2590// server to finish authentication, unlike LOGIN which is just a single
2591// username/password.
2593// We may already have ambient TLS credentials that have not been activated.
2595// Status: Not authenticated.
2596func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
2600 // For many failed auth attempts, slow down verification attempts.
2601 if c.authFailed > 3 && authFailDelay > 0 {
2602 mox.Sleep(mox.Context, time.Duration(c.authFailed-3)*authFailDelay)
2605 // If authentication fails due to missing derived secrets, we don't hold it against
2606 // the connection. There is no way to indicate server support for an authentication
2607 // mechanism, but that a mechanism won't work for an account.
2608 var missingDerivedSecrets bool
2610 c.authFailed++ // Compensated on success.
2612 if missingDerivedSecrets {
2615 // On the 3rd failed authentication, start responding slowly. Successful auth will
2616 // cause fast responses again.
2617 if c.authFailed >= 3 {
2622 c.newLoginAttempt(true, "")
2624 if c.loginAttempt.Result == store.AuthSuccess {
2625 mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
2626 } else if !missingDerivedSecrets {
2627 mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
2633 authType := p.xatom()
2635 xreadInitial := func() []byte {
2639 line = c.xreadline(false)
2643 line = p.remainder()
2646 line = "" // Base64 decode will result in empty buffer.
2651 c.loginAttempt.Result = store.AuthAborted
2652 xsyntaxErrorf("authenticate aborted by client")
2654 buf, err := base64.StdEncoding.DecodeString(line)
2656 xsyntaxErrorf("parsing base64: %v", err)
2661 xreadContinuation := func() []byte {
2662 line := c.xreadline(false)
2664 c.loginAttempt.Result = store.AuthAborted
2665 xsyntaxErrorf("authenticate aborted by client")
2667 buf, err := base64.StdEncoding.DecodeString(line)
2669 xsyntaxErrorf("parsing base64: %v", err)
2674 // The various authentication mechanisms set account and username. We may already
2675 // have an account and username from TLS client authentication. Afterwards, we
2676 // check that the account is the same.
2677 var account *store.Account
2681 err := account.Close()
2682 c.xsanity(err, "close account")
2686 switch strings.ToUpper(authType) {
2688 c.loginAttempt.AuthMech = "plain"
2690 if !c.noRequireSTARTTLS && !c.tls {
2692 xusercodeErrorf("PRIVACYREQUIRED", "tls required for login")
2695 // Plain text passwords, mark as traceauth.
2696 defer c.xtraceread(mlog.LevelTraceauth)()
2697 buf := xreadInitial()
2698 c.xtraceread(mlog.LevelTrace) // Restore.
2699 plain := bytes.Split(buf, []byte{0})
2700 if len(plain) != 3 {
2701 xsyntaxErrorf("bad plain auth data, expected 3 nul-separated tokens, got %d tokens", len(plain))
2703 authz := norm.NFC.String(string(plain[0]))
2704 username = norm.NFC.String(string(plain[1]))
2705 password := string(plain[2])
2706 c.loginAttempt.LoginAddress = username
2708 if authz != "" && authz != username {
2709 xusercodeErrorf("AUTHORIZATIONFAILED", "cannot assume role")
2713 account, c.loginAttempt.AccountName, err = store.OpenEmailAuth(c.log, username, password, false)
2715 if errors.Is(err, store.ErrUnknownCredentials) {
2716 c.loginAttempt.Result = store.AuthBadCredentials
2717 c.log.Info("authentication failed", slog.String("username", username))
2718 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2720 xusercodeErrorf("", "error")
2724 c.loginAttempt.AuthMech = strings.ToLower(authType)
2730 chal := fmt.Sprintf("<%d.%d@%s>", uint64(mox.CryptoRandInt()), time.Now().UnixNano(), mox.Conf.Static.HostnameDomain.ASCII)
2731 c.xwritelinef("+ %s", base64.StdEncoding.EncodeToString([]byte(chal)))
2733 resp := xreadContinuation()
2734 t := strings.Split(string(resp), " ")
2735 if len(t) != 2 || len(t[1]) != 2*md5.Size {
2736 xsyntaxErrorf("malformed cram-md5 response")
2738 username = norm.NFC.String(t[0])
2739 c.loginAttempt.LoginAddress = username
2740 c.log.Debug("cram-md5 auth", slog.String("address", username))
2742 account, c.loginAttempt.AccountName, _, err = store.OpenEmail(c.log, username, false)
2744 if errors.Is(err, store.ErrUnknownCredentials) {
2745 c.loginAttempt.Result = store.AuthBadCredentials
2746 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2747 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2749 xserverErrorf("looking up address: %v", err)
2751 var ipadhash, opadhash hash.Hash
2752 account.WithRLock(func() {
2753 err := account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
2754 password, err := bstore.QueryTx[store.Password](tx).Get()
2755 if err == bstore.ErrAbsent {
2756 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2757 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2763 ipadhash = password.CRAMMD5.Ipad
2764 opadhash = password.CRAMMD5.Opad
2767 xcheckf(err, "tx read")
2769 if ipadhash == nil || opadhash == nil {
2770 c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", slog.String("username", username))
2771 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2772 missingDerivedSecrets = true
2773 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2777 ipadhash.Write([]byte(chal))
2778 opadhash.Write(ipadhash.Sum(nil))
2779 digest := fmt.Sprintf("%x", opadhash.Sum(nil))
2781 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2782 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2785 case "SCRAM-SHA-256-PLUS", "SCRAM-SHA-256", "SCRAM-SHA-1-PLUS", "SCRAM-SHA-1":
2786 // todo: improve handling of errors during scram. e.g. invalid parameters. should we abort the imap command, or continue until the end and respond with a scram-level error?
2787 // todo: use single implementation between ../imapserver/server.go and ../smtpserver/server.go
2789 // No plaintext credentials, we can log these normally.
2791 c.loginAttempt.AuthMech = strings.ToLower(authType)
2792 var h func() hash.Hash
2793 switch c.loginAttempt.AuthMech {
2794 case "scram-sha-1", "scram-sha-1-plus":
2796 case "scram-sha-256", "scram-sha-256-plus":
2799 xserverErrorf("missing case for scram variant")
2802 var cs *tls.ConnectionState
2803 requireChannelBinding := strings.HasSuffix(c.loginAttempt.AuthMech, "-plus")
2804 if requireChannelBinding && !c.tls {
2805 xuserErrorf("cannot use plus variant with tls channel binding without tls")
2808 xcs := c.conn.(*tls.Conn).ConnectionState()
2811 c0 := xreadInitial()
2812 ss, err := scram.NewServer(h, c0, cs, requireChannelBinding)
2814 c.log.Infox("scram protocol error", err, slog.Any("remote", c.remoteIP))
2815 xuserErrorf("scram protocol error: %s", err)
2817 username = ss.Authentication
2818 c.loginAttempt.LoginAddress = username
2819 c.log.Debug("scram auth", slog.String("authentication", username))
2820 // We check for login being disabled when finishing.
2821 account, c.loginAttempt.AccountName, _, err = store.OpenEmail(c.log, username, false)
2823 // todo: we could continue scram with a generated salt, deterministically generated
2824 // from the username. that way we don't have to store anything but attackers cannot
2825 // learn if an account exists. same for absent scram saltedpassword below.
2826 xuserErrorf("scram not possible")
2828 if ss.Authorization != "" && ss.Authorization != username {
2829 xuserErrorf("authentication with authorization for different user not supported")
2831 var xscram store.SCRAM
2832 account.WithRLock(func() {
2833 err := account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
2834 password, err := bstore.QueryTx[store.Password](tx).Get()
2835 if err == bstore.ErrAbsent {
2836 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2837 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2839 xcheckf(err, "fetching credentials")
2840 switch c.loginAttempt.AuthMech {
2841 case "scram-sha-1", "scram-sha-1-plus":
2842 xscram = password.SCRAMSHA1
2843 case "scram-sha-256", "scram-sha-256-plus":
2844 xscram = password.SCRAMSHA256
2846 xserverErrorf("missing case for scram credentials")
2848 if len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0 {
2849 missingDerivedSecrets = true
2850 c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", slog.String("username", username))
2851 xuserErrorf("scram not possible")
2855 xcheckf(err, "read tx")
2857 s1, err := ss.ServerFirst(xscram.Iterations, xscram.Salt)
2858 xcheckf(err, "scram first server step")
2859 c.xwritelinef("+ %s", base64.StdEncoding.EncodeToString([]byte(s1)))
2860 c2 := xreadContinuation()
2861 s3, err := ss.Finish(c2, xscram.SaltedPassword)
2863 c.xwritelinef("+ %s", base64.StdEncoding.EncodeToString([]byte(s3)))
2866 c.xreadline(false) // Should be "*" for cancellation.
2867 if errors.Is(err, scram.ErrInvalidProof) {
2868 c.loginAttempt.Result = store.AuthBadCredentials
2869 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2870 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2871 } else if errors.Is(err, scram.ErrChannelBindingsDontMatch) {
2872 c.loginAttempt.Result = store.AuthBadChannelBinding
2873 c.log.Warn("bad channel binding during authentication, potential mitm", slog.String("username", username), slog.Any("remote", c.remoteIP))
2874 xusercodeErrorf("AUTHENTICATIONFAILED", "channel bindings do not match, potential mitm")
2875 } else if errors.Is(err, scram.ErrInvalidEncoding) {
2876 c.loginAttempt.Result = store.AuthBadProtocol
2877 c.log.Infox("bad scram protocol message", err, slog.String("username", username), slog.Any("remote", c.remoteIP))
2878 xuserErrorf("bad scram protocol message: %s", err)
2880 xuserErrorf("server final: %w", err)
2884 // The message should be empty. todo: should we require it is empty?
2888 c.loginAttempt.AuthMech = "external"
2891 buf := xreadInitial()
2892 username = norm.NFC.String(string(buf))
2893 c.loginAttempt.LoginAddress = username
2896 xusercodeErrorf("AUTHENTICATIONFAILED", "tls required for tls client certificate authentication")
2898 if c.account == nil {
2899 xusercodeErrorf("AUTHENTICATIONFAILED", "missing client certificate, required for tls client certificate authentication")
2903 username = c.username
2904 c.loginAttempt.LoginAddress = username
2907 account, c.loginAttempt.AccountName, _, err = store.OpenEmail(c.log, username, false)
2908 xcheckf(err, "looking up username from tls client authentication")
2911 c.loginAttempt.AuthMech = "(unrecognized)"
2912 xuserErrorf("method not supported")
2915 if accConf, ok := account.Conf(); !ok {
2916 xserverErrorf("cannot get account config")
2917 } else if accConf.LoginDisabled != "" {
2918 c.loginAttempt.Result = store.AuthLoginDisabled
2919 c.log.Info("account login disabled", slog.String("username", username))
2920 // No AUTHENTICATIONFAILED code, clients could prompt users for different password.
2921 xuserErrorf("%w: %s", store.ErrLoginDisabled, accConf.LoginDisabled)
2924 // We may already have TLS credentials. They won't have been enabled, or we could
2925 // get here due to the state machine that doesn't allow authentication while being
2926 // authenticated. But allow another SASL authentication, but it has to be for the
2927 // same account. It can be for a different username (email address) of the account.
2928 if c.account != nil {
2929 if account != c.account {
2930 c.log.Debug("sasl authentication for different account than tls client authentication, aborting connection",
2931 slog.String("saslmechanism", c.loginAttempt.AuthMech),
2932 slog.String("saslaccount", account.Name),
2933 slog.String("tlsaccount", c.account.Name),
2934 slog.String("saslusername", username),
2935 slog.String("tlsusername", c.username),
2937 xusercodeErrorf("AUTHENTICATIONFAILED", "authentication failed, tls client certificate public key belongs to another account")
2938 } else if username != c.username {
2939 c.log.Debug("sasl authentication for different username than tls client certificate authentication, switching to sasl username",
2940 slog.String("saslmechanism", c.loginAttempt.AuthMech),
2941 slog.String("saslusername", username),
2942 slog.String("tlsusername", c.username),
2943 slog.String("account", c.account.Name),
2948 account = nil // Prevent cleanup.
2950 c.username = username
2952 c.comm = store.RegisterComm(c.account)
2956 c.loginAttempt.AccountName = c.account.Name
2957 c.loginAttempt.LoginAddress = c.username
2958 c.loginAttempt.Result = store.AuthSuccess
2960 c.state = stateAuthenticated
2961 c.xwriteresultf("%s OK [CAPABILITY %s] authenticate done", tag, c.capabilities())
2964// Login logs in with username and password.
2966// Status: Not authenticated.
2967func (c *conn) cmdLogin(tag, cmd string, p *parser) {
2970 c.newLoginAttempt(true, "login")
2972 if c.loginAttempt.Result == store.AuthSuccess {
2973 mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
2975 mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
2979 // todo: get this line logged with traceauth. the plaintext password is included on the command line, which we've already read (before dispatching to this function).
2983 username := norm.NFC.String(p.xastring())
2984 c.loginAttempt.LoginAddress = username
2986 password := p.xastring()
2989 if !c.noRequireSTARTTLS && !c.tls {
2991 xusercodeErrorf("PRIVACYREQUIRED", "tls required for login")
2994 // For many failed auth attempts, slow down verification attempts.
2995 if c.authFailed > 3 && authFailDelay > 0 {
2996 mox.Sleep(mox.Context, time.Duration(c.authFailed-3)*authFailDelay)
2998 c.authFailed++ // Compensated on success.
3000 // On the 3rd failed authentication, start responding slowly. Successful auth will
3001 // cause fast responses again.
3002 if c.authFailed >= 3 {
3007 account, accName, err := store.OpenEmailAuth(c.log, username, password, true)
3008 c.loginAttempt.AccountName = accName
3011 if errors.Is(err, store.ErrUnknownCredentials) {
3012 c.loginAttempt.Result = store.AuthBadCredentials
3013 code = "AUTHENTICATIONFAILED"
3014 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
3015 } else if errors.Is(err, store.ErrLoginDisabled) {
3016 c.loginAttempt.Result = store.AuthLoginDisabled
3017 c.log.Info("account login disabled", slog.String("username", username))
3018 // There is no specific code for "account disabled" in IMAP. AUTHORIZATIONFAILED is
3019 // not a good idea, it will prompt users for a password. ALERT seems reasonable,
3020 // but may cause email clients to suppress the message since we are not yet
3022 xuserErrorf("%s", err)
3024 xusercodeErrorf(code, "login failed")
3028 err := account.Close()
3029 c.xsanity(err, "close account")
3033 // We may already have TLS credentials. They won't have been enabled, or we could
3034 // get here due to the state machine that doesn't allow authentication while being
3035 // authenticated. But allow another SASL authentication, but it has to be for the
3036 // same account. It can be for a different username (email address) of the account.
3037 if c.account != nil {
3038 if account != c.account {
3039 c.log.Debug("sasl authentication for different account than tls client authentication, aborting connection",
3040 slog.String("saslmechanism", "login"),
3041 slog.String("saslaccount", account.Name),
3042 slog.String("tlsaccount", c.account.Name),
3043 slog.String("saslusername", username),
3044 slog.String("tlsusername", c.username),
3046 xusercodeErrorf("AUTHENTICATIONFAILED", "authentication failed, tls client certificate public key belongs to another account")
3047 } else if username != c.username {
3048 c.log.Debug("sasl authentication for different username than tls client certificate authentication, switching to sasl username",
3049 slog.String("saslmechanism", "login"),
3050 slog.String("saslusername", username),
3051 slog.String("tlsusername", c.username),
3052 slog.String("account", c.account.Name),
3057 account = nil // Prevent cleanup.
3059 c.username = username
3061 c.comm = store.RegisterComm(c.account)
3063 c.loginAttempt.LoginAddress = c.username
3064 c.loginAttempt.AccountName = c.account.Name
3065 c.loginAttempt.Result = store.AuthSuccess
3068 c.state = stateAuthenticated
3069 c.xwriteresultf("%s OK [CAPABILITY %s] login done", tag, c.capabilities())
3072// Enable explicitly opts in to an extension. A server can typically send new kinds
3073// of responses to a client. Most extensions do not require an ENABLE because a
3074// client implicitly opts in to new response syntax by making a requests that uses
3075// new optional extension request syntax.
3077// State: Authenticated and selected.
3078func (c *conn) cmdEnable(tag, cmd string, p *parser) {
3084 caps := []string{p.xatom()}
3087 caps = append(caps, p.xatom())
3090 // Clients should only send capabilities that need enabling.
3091 // We should only echo that we recognize as needing enabling.
3092 var enabled strings.Builder
3095 // Accounts can suppress capabilities, we ignore them when the client tries to
3097 var disabled []string
3098 if c.account != nil {
3099 conf, _ := c.account.Conf()
3100 disabled = conf.IMAPCapabilitiesDisabled
3103 for _, s := range caps {
3104 cap := capability(strings.ToUpper(s))
3106 if slices.Contains(disabled, string(cap)) {
3114 c.enabled[cap] = true
3115 enabled.WriteString(" " + s)
3117 c.enabled[cap] = true
3118 enabled.WriteString(" " + s)
3121 c.enabled[cap] = true
3122 enabled.WriteString(" " + s)
3124 c.enabled[cap] = true
3125 enabled.WriteString(" " + s)
3131 if qresync && !c.enabled[capCondstore] {
3132 c.xensureCondstore(nil)
3133 enabled.WriteString(" CONDSTORE")
3137 c.xbwritelinef("* ENABLED%s", enabled.String())
3142// If a mailbox is selected, an untagged OK with HIGHESTMODSEQ is written to the
3143// client. If tx is non-nil, it is used to read the HIGHESTMODSEQ from the
3144// database. Otherwise a new read-only transaction is created.
3145func (c *conn) xensureCondstore(tx *bstore.Tx) {
3146 if !c.enabled[capCondstore] {
3147 c.enabled[capCondstore] = true
3148 // todo spec: can we send an untagged enabled response?
3150 if c.mailboxID <= 0 {
3154 var mb store.Mailbox
3156 c.xdbread(func(tx *bstore.Tx) {
3157 mb = c.xmailboxID(tx, c.mailboxID)
3160 mb = c.xmailboxID(tx, c.mailboxID)
3162 c.xbwritelinef("* OK [HIGHESTMODSEQ %d] after condstore-enabling command", mb.ModSeq.Client())
3166// State: Authenticated and selected.
3167func (c *conn) cmdSelect(tag, cmd string, p *parser) {
3168 c.cmdSelectExamine(true, tag, cmd, p)
3171// State: Authenticated and selected.
3172func (c *conn) cmdExamine(tag, cmd string, p *parser) {
3173 c.cmdSelectExamine(false, tag, cmd, p)
3176// Select and examine are almost the same commands. Select just opens a mailbox for
3177// read/write and examine opens a mailbox readonly.
3179// State: Authenticated and selected.
3180func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) {
3188 name := p.xmailbox()
3190 var qruidvalidity uint32
3191 var qrmodseq int64 // QRESYNC required parameters.
3192 var qrknownUIDs, qrknownSeqSet, qrknownUIDSet *numSet // QRESYNC optional parameters.
3194 seen := map[string]bool{}
3196 for len(seen) == 0 || !p.take(")") {
3197 w := p.xtakelist("CONDSTORE", "QRESYNC")
3199 xsyntaxErrorf("duplicate select parameter %s", w)
3209 // Note: unlike with CONDSTORE, there are no QRESYNC-related commands/parameters
3210 // that enable capabilities.
3211 if !c.enabled[capQresync] {
3213 xsyntaxErrorf("QRESYNC must first be enabled")
3219 qrmodseq = p.xnznumber64()
3221 seqMatchData := p.take("(")
3225 seqMatchData = p.take(" (")
3228 ss0 := p.xnumSet0(false, false)
3229 qrknownSeqSet = &ss0
3231 ss1 := p.xnumSet0(false, false)
3232 qrknownUIDSet = &ss1
3238 panic("missing case for select param " + w)
3244 // Deselect before attempting the new select. This means we will deselect when an
3245 // error occurs during select.
3247 if c.state == stateSelected {
3249 c.xbwritelinef("* OK [CLOSED] x")
3253 if c.uidonly && qrknownSeqSet != nil {
3255 xsyntaxCodeErrorf("UIDREQUIRED", "cannot use message sequence match data with uidonly enabled")
3258 name = xcheckmailboxname(name, true)
3260 var mb store.Mailbox
3261 c.account.WithRLock(func() {
3262 // Flush pending changes. We are about to fetch the current state from the
3263 // database. We don't want appends that were pending since the beginning of the
3264 // connection to be processed after we've read the latest state from the database,
3265 // e.g. by a status commands after select. It would mean we would try to apply
3266 // changes that we already have in our connection state.
3267 overflow, pendingChanges := c.comm.Get()
3268 c.xapplyChanges(overflow, pendingChanges, false)
3270 c.xdbread(func(tx *bstore.Tx) {
3271 mb = c.xmailbox(tx, name, "")
3273 var firstUnseen msgseq = 0
3275 c.uidnext = mb.UIDNext
3277 c.exists = uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
3279 c.uids = []store.UID{}
3281 q := bstore.QueryTx[store.Message](tx)
3282 q.FilterNonzero(store.Message{MailboxID: mb.ID})
3283 q.FilterEqual("Expunged", false)
3285 err := q.ForEach(func(m store.Message) error {
3286 c.uids = append(c.uids, m.UID)
3287 if firstUnseen == 0 && !m.Seen {
3288 firstUnseen = msgseq(len(c.uids))
3292 xcheckf(err, "fetching uids")
3294 c.exists = uint32(len(c.uids))
3298 if len(mb.Keywords) > 0 {
3299 flags = " " + strings.Join(mb.Keywords, " ")
3301 c.xbwritelinef(`* FLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent%s)`, flags)
3302 c.xbwritelinef(`* OK [PERMANENTFLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*)] x`)
3303 if !c.enabled[capIMAP4rev2] {
3304 c.xbwritelinef(`* 0 RECENT`)
3306 c.xbwritelinef(`* %d EXISTS`, c.exists)
3307 if !c.enabled[capIMAP4rev2] && firstUnseen > 0 {
3309 c.xbwritelinef(`* OK [UNSEEN %d] x`, firstUnseen)
3311 c.xbwritelinef(`* OK [UIDVALIDITY %d] x`, mb.UIDValidity)
3312 c.xbwritelinef(`* OK [UIDNEXT %d] x`, mb.UIDNext)
3313 c.xbwritelinef(`* LIST () "/" %s`, mailboxt(mb.Name).pack(c))
3314 if c.enabled[capCondstore] {
3317 c.xbwritelinef(`* OK [HIGHESTMODSEQ %d] x`, mb.ModSeq.Client())
3321 if qruidvalidity == mb.UIDValidity {
3322 // We send the vanished UIDs at the end, so we can easily combine the modseq
3323 // changes and vanished UIDs that result from that, with the vanished UIDs from the
3324 // case where we don't store enough history.
3325 vanishedUIDs := map[store.UID]struct{}{}
3327 var preVanished store.UID
3328 var oldClientUID store.UID
3329 // If samples of known msgseq and uid pairs are given (they must be in order), we
3330 // use them to determine the earliest UID for which we send VANISHED responses.
3332 if qrknownSeqSet != nil {
3333 if !qrknownSeqSet.isBasicIncreasing() {
3334 xuserErrorf("QRESYNC known message sequence set must be numeric and strictly increasing")
3336 if !qrknownUIDSet.isBasicIncreasing() {
3337 xuserErrorf("QRESYNC known uid set must be numeric and strictly increasing")
3339 seqiter := qrknownSeqSet.newIter()
3340 uiditer := qrknownUIDSet.newIter()
3342 msgseq, ok0 := seqiter.Next()
3343 uid, ok1 := uiditer.Next()
3346 } else if !ok0 || !ok1 {
3347 xsyntaxErrorf("invalid combination of known sequence set and uid set, must be of equal length")
3349 i := int(msgseq - 1)
3350 // Access to c.uids is safe, qrknownSeqSet and uidonly cannot both be set.
3351 if i < 0 || i >= int(c.exists) || c.uids[i] != store.UID(uid) {
3352 if uidSearch(c.uids, store.UID(uid)) <= 0 {
3353 // We will check this old client UID for consistency below.
3354 oldClientUID = store.UID(uid)
3358 preVanished = store.UID(uid + 1)
3362 // We gather vanished UIDs and report them at the end. This seems OK because we
3363 // already sent HIGHESTMODSEQ, and a client should know not to commit that value
3364 // until after it has seen the tagged OK of this command. The RFC has a remark
3365 // about ordering of some untagged responses, it's not immediately clear what it
3366 // means, but given the examples appears to allude to servers that decide to not
3367 // send expunge/vanished before the tagged OK.
3370 if oldClientUID > 0 {
3371 // The client sent a UID that is now removed. This is typically fine. But we check
3372 // that it is consistent with the modseq the client sent. If the UID already didn't
3373 // exist at that modseq, the client may be missing some information.
3374 q := bstore.QueryTx[store.Message](tx)
3375 q.FilterNonzero(store.Message{MailboxID: mb.ID, UID: oldClientUID})
3378 // If client claims to be up to date up to and including qrmodseq, and the message
3379 // was deleted at or before that time, we send changes from just before that
3380 // modseq, and we send vanished for all UIDs.
3381 if m.Expunged && qrmodseq >= m.ModSeq.Client() {
3382 qrmodseq = m.ModSeq.Client() - 1
3385 c.xbwritelinef("* OK [ALERT] Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full synchronization recommended.")
3387 } else if err != bstore.ErrAbsent {
3388 xcheckf(err, "checking old client uid")
3392 q := bstore.QueryTx[store.Message](tx)
3393 q.FilterNonzero(store.Message{MailboxID: mb.ID})
3394 // Note: we don't filter by Expunged.
3395 q.FilterGreater("ModSeq", store.ModSeqFromClient(qrmodseq))
3396 q.FilterLessEqual("ModSeq", mb.ModSeq)
3397 q.FilterLess("UID", c.uidnext)
3399 err := q.ForEach(func(m store.Message) error {
3400 if m.Expunged && m.UID < preVanished {
3404 if qrknownUIDs != nil && !qrknownUIDs.contains(uint32(m.UID)) {
3408 vanishedUIDs[m.UID] = struct{}{}
3413 c.xbwritelinef("* %d UIDFETCH (FLAGS %s MODSEQ (%d))", m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
3414 } else if msgseq := c.sequence(m.UID); msgseq > 0 {
3415 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s MODSEQ (%d))", msgseq, m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
3419 xcheckf(err, "listing changed messages")
3421 highDeletedModSeq, err := c.account.HighestDeletedModSeq(tx)
3422 xcheckf(err, "getting highest deleted modseq")
3424 // If we don't have enough history, we go through all UIDs and look them up, and
3425 // add them to the vanished list if they have disappeared.
3426 if qrmodseq < highDeletedModSeq.Client() {
3427 // If no "known uid set" was in the request, we substitute 1:max or the empty set.
3429 if qrknownUIDs == nil {
3430 qrknownUIDs = &numSet{ranges: []numRange{{first: setNumber{number: 1}, last: &setNumber{number: uint32(c.uidnext - 1)}}}}
3434 // note: qrknownUIDs will not contain "*".
3435 for _, r := range qrknownUIDs.xinterpretStar(func() store.UID { return 0 }).ranges {
3436 // Gather UIDs for this range.
3437 var uids []store.UID
3438 q := bstore.QueryTx[store.Message](tx)
3439 q.FilterNonzero(store.Message{MailboxID: mb.ID})
3440 q.FilterEqual("Expunged", false)
3442 q.FilterEqual("UID", r.first.number)
3444 q.FilterGreaterEqual("UID", r.first.number)
3445 q.FilterLessEqual("UID", r.last.number)
3448 for m, err := range q.All() {
3449 xcheckf(err, "enumerating uids")
3450 uids = append(uids, m.UID)
3453 // Find UIDs missing from the database.
3456 uid, ok := iter.Next()
3460 if uidSearch(uids, store.UID(uid)) <= 0 {
3461 vanishedUIDs[store.UID(uid)] = struct{}{}
3466 // Ensure it is in ascending order, no needless first/last ranges. qrknownUIDs cannot contain a star.
3467 iter := qrknownUIDs.newIter()
3469 v, ok := iter.Next()
3473 if c.sequence(store.UID(v)) <= 0 {
3474 vanishedUIDs[store.UID(v)] = struct{}{}
3480 // Now that we have all vanished UIDs, send them over compactly.
3481 if len(vanishedUIDs) > 0 {
3482 l := slices.Sorted(maps.Keys(vanishedUIDs))
3484 for _, s := range compactUIDSet(l).Strings(4*1024 - 32) {
3485 c.xbwritelinef("* VANISHED (EARLIER) %s", s)
3493 c.xbwriteresultf("%s OK [READ-WRITE] x", tag)
3496 c.xbwriteresultf("%s OK [READ-ONLY] x", tag)
3500 c.state = stateSelected
3501 c.searchResult = nil
3505// Create makes a new mailbox, and its parents too if absent.
3507// State: Authenticated and selected.
3508func (c *conn) cmdCreate(tag, cmd string, p *parser) {
3514 name := p.xmailbox()
3516 var useAttrs []string // Special-use attributes without leading \.
3519 // We only support "USE", and there don't appear to be more types of parameters.
3524 useAttrs = append(useAttrs, p.xatom())
3540 name = xcheckmailboxname(name, false)
3542 var specialUse store.SpecialUse
3543 specialUseBools := map[string]*bool{
3544 "archive": &specialUse.Archive,
3545 "drafts": &specialUse.Draft,
3546 "junk": &specialUse.Junk,
3547 "sent": &specialUse.Sent,
3548 "trash": &specialUse.Trash,
3550 for _, s := range useAttrs {
3551 p, ok := specialUseBools[strings.ToLower(s)]
3554 xusercodeErrorf("USEATTR", `cannot create mailbox with special-use attribute \%s`, s)
3559 var changes []store.Change
3560 var created []string // Created mailbox names.
3562 c.account.WithWLock(func() {
3563 c.xdbwrite(func(tx *bstore.Tx) {
3566 _, changes, created, exists, err = c.account.MailboxCreate(tx, name, specialUse)
3569 xuserErrorf("mailbox already exists")
3571 xcheckf(err, "creating mailbox")
3574 c.broadcast(changes)
3577 for _, n := range created {
3580 if c.enabled[capIMAP4rev2] && n == name && name != origName && !(name == "Inbox" || strings.HasPrefix(name, "Inbox/")) {
3581 oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(origName).pack(c))
3583 c.xbwritelinef(`* LIST (\Subscribed) "/" %s%s`, mailboxt(n).pack(c), oldname)
3588// Delete removes a mailbox and all its messages and annotations.
3589// Inbox cannot be removed.
3591// State: Authenticated and selected.
3592func (c *conn) cmdDelete(tag, cmd string, p *parser) {
3598 name := p.xmailbox()
3601 name = xcheckmailboxname(name, false)
3603 c.account.WithWLock(func() {
3604 var mb store.Mailbox
3605 var changes []store.Change
3607 c.xdbwrite(func(tx *bstore.Tx) {
3608 mb = c.xmailbox(tx, name, "NONEXISTENT")
3610 var hasChildren bool
3612 changes, hasChildren, err = c.account.MailboxDelete(context.TODO(), c.log, tx, &mb)
3614 xusercodeErrorf("HASCHILDREN", "mailbox has a child, only leaf mailboxes can be deleted")
3616 xcheckf(err, "deleting mailbox")
3619 c.broadcast(changes)
3625// Rename changes the name of a mailbox.
3626// Renaming INBOX is special, it moves the inbox messages to a new mailbox, leaving
3627// inbox empty, but copying metadata annotations.
3628// Renaming a mailbox with submailboxes also renames all submailboxes.
3629// Subscriptions stay with the old name, though newly created missing parent
3630// mailboxes for the destination name are automatically subscribed.
3632// State: Authenticated and selected.
3633func (c *conn) cmdRename(tag, cmd string, p *parser) {
3644 src = xcheckmailboxname(src, true)
3645 dst = xcheckmailboxname(dst, false)
3647 var cleanupIDs []int64
3649 for _, id := range cleanupIDs {
3650 p := c.account.MessagePath(id)
3652 c.xsanity(err, "cleaning up message")
3656 c.account.WithWLock(func() {
3657 var changes []store.Change
3659 c.xdbwrite(func(tx *bstore.Tx) {
3660 mbSrc := c.xmailbox(tx, src, "NONEXISTENT")
3662 // Handle common/simple case first.
3664 var modseq store.ModSeq
3665 var alreadyExists bool
3667 changes, _, alreadyExists, err = c.account.MailboxRename(tx, &mbSrc, dst, &modseq)
3669 xusercodeErrorf("ALREADYEXISTS", "%s", err)
3671 xcheckf(err, "renaming mailbox")
3675 // Inbox is very special. Unlike other mailboxes, its children are not moved. And
3676 // unlike a regular move, its messages are moved to a newly created mailbox. We do
3677 // indeed create a new destination mailbox and actually move the messages.
3679 exists, err := c.account.MailboxExists(tx, dst)
3680 xcheckf(err, "checking if destination mailbox exists")
3682 xusercodeErrorf("ALREADYEXISTS", "destination mailbox %q already exists", dst)
3685 xuserErrorf("cannot move inbox to itself")
3688 var modseq store.ModSeq
3689 mbDst, chl, err := c.account.MailboxEnsure(tx, dst, false, store.SpecialUse{}, &modseq)
3690 xcheckf(err, "creating destination mailbox")
3694 qa := bstore.QueryTx[store.Annotation](tx)
3695 qa.FilterNonzero(store.Annotation{MailboxID: mbSrc.ID})
3696 qa.FilterEqual("Expunged", false)
3697 annotations, err := qa.List()
3698 xcheckf(err, "get annotations to copy for inbox")
3699 for _, a := range annotations {
3701 a.MailboxID = mbDst.ID
3703 a.CreateSeq = modseq
3704 err := tx.Insert(&a)
3705 xcheckf(err, "copy annotation to destination mailbox")
3706 changes = append(changes, a.Change(mbDst.Name))
3708 c.xcheckMetadataSize(tx)
3710 // Build query that selects messages to move.
3711 q := bstore.QueryTx[store.Message](tx)
3712 q.FilterNonzero(store.Message{MailboxID: mbSrc.ID})
3713 q.FilterEqual("Expunged", false)
3716 newIDs, chl := c.xmoveMessages(tx, q, 0, modseq, &mbSrc, &mbDst)
3717 changes = append(changes, chl...)
3723 c.broadcast(changes)
3729// Subscribe marks a mailbox path as subscribed. The mailbox does not have to
3730// exist. Subscribed may mean an email client will show the mailbox in its UI
3731// and/or periodically fetch new messages for the mailbox.
3733// State: Authenticated and selected.
3734func (c *conn) cmdSubscribe(tag, cmd string, p *parser) {
3740 name := p.xmailbox()
3743 name = xcheckmailboxname(name, true)
3745 c.account.WithWLock(func() {
3746 var changes []store.Change
3748 c.xdbwrite(func(tx *bstore.Tx) {
3750 changes, err = c.account.SubscriptionEnsure(tx, name)
3751 xcheckf(err, "ensuring subscription")
3754 c.broadcast(changes)
3760// Unsubscribe marks a mailbox as not subscribed. The mailbox doesn't have to exist.
3762// State: Authenticated and selected.
3763func (c *conn) cmdUnsubscribe(tag, cmd string, p *parser) {
3769 name := p.xmailbox()
3772 name = xcheckmailboxname(name, true)
3774 c.account.WithWLock(func() {
3775 var changes []store.Change
3777 c.xdbwrite(func(tx *bstore.Tx) {
3779 err := tx.Delete(&store.Subscription{Name: name})
3780 if err == bstore.ErrAbsent {
3781 exists, err := c.account.MailboxExists(tx, name)
3782 xcheckf(err, "checking if mailbox exists")
3784 xuserErrorf("mailbox does not exist")
3788 xcheckf(err, "removing subscription")
3791 exists, err := c.account.MailboxExists(tx, name)
3792 xcheckf(err, "looking up mailbox existence")
3794 flags = []string{`\NonExistent`}
3797 changes = []store.Change{store.ChangeRemoveSubscription{MailboxName: name, ListFlags: flags}}
3800 c.broadcast(changes)
3802 // todo: can we send untagged message about a mailbox no longer being subscribed?
3808// LSUB command for listing subscribed mailboxes.
3809// Removed in IMAP4rev2, only in IMAP4rev1.
3811// State: Authenticated and selected.
3812func (c *conn) cmdLsub(tag, cmd string, p *parser) {
3820 pattern := p.xlistMailbox()
3823 re := xmailboxPatternMatcher(ref, []string{pattern})
3826 c.xdbread(func(tx *bstore.Tx) {
3827 q := bstore.QueryTx[store.Subscription](tx)
3829 subscriptions, err := q.List()
3830 xcheckf(err, "querying subscriptions")
3832 have := map[string]bool{}
3833 subscribedKids := map[string]bool{}
3834 ispercent := strings.HasSuffix(pattern, "%")
3835 for _, sub := range subscriptions {
3838 for p := mox.ParentMailboxName(name); p != ""; p = mox.ParentMailboxName(p) {
3839 subscribedKids[p] = true
3842 if !re.MatchString(name) {
3846 line := fmt.Sprintf(`* LSUB () "/" %s`, mailboxt(name).pack(c))
3847 lines = append(lines, line)
3855 qmb := bstore.QueryTx[store.Mailbox](tx)
3856 qmb.FilterEqual("Expunged", false)
3858 err = qmb.ForEach(func(mb store.Mailbox) error {
3859 if have[mb.Name] || !subscribedKids[mb.Name] || !re.MatchString(mb.Name) {
3862 line := fmt.Sprintf(`* LSUB (\NoSelect) "/" %s`, mailboxt(mb.Name).pack(c))
3863 lines = append(lines, line)
3866 xcheckf(err, "querying mailboxes")
3870 for _, line := range lines {
3871 c.xbwritelinef("%s", line)
3876// The namespace command returns the mailbox path separator. We only implement
3877// the personal mailbox hierarchy, no shared/other.
3879// In IMAP4rev2, it was an extension before.
3881// State: Authenticated and selected.
3882func (c *conn) cmdNamespace(tag, cmd string, p *parser) {
3889 c.xbwritelinef(`* NAMESPACE (("" "/")) NIL NIL`)
3893// The status command returns information about a mailbox, such as the number of
3894// messages, "uid validity", etc. Nowadays, the extended LIST command can return
3895// the same information about many mailboxes for one command.
3897// State: Authenticated and selected.
3898func (c *conn) cmdStatus(tag, cmd string, p *parser) {
3904 name := p.xmailbox()
3907 attrs := []string{p.xstatusAtt()}
3910 attrs = append(attrs, p.xstatusAtt())
3914 name = xcheckmailboxname(name, true)
3916 var mb store.Mailbox
3918 var responseLine string
3919 c.account.WithRLock(func() {
3920 c.xdbread(func(tx *bstore.Tx) {
3921 mb = c.xmailbox(tx, name, "")
3922 responseLine = c.xstatusLine(tx, mb, attrs)
3926 c.xbwritelinef("%s", responseLine)
3931func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) string {
3932 status := []string{}
3933 for _, a := range attrs {
3934 A := strings.ToUpper(a)
3937 status = append(status, A, fmt.Sprintf("%d", mb.Total+mb.Deleted))
3939 status = append(status, A, fmt.Sprintf("%d", mb.UIDNext))
3941 status = append(status, A, fmt.Sprintf("%d", mb.UIDValidity))
3943 status = append(status, A, fmt.Sprintf("%d", mb.Unseen))
3945 status = append(status, A, fmt.Sprintf("%d", mb.Deleted))
3947 status = append(status, A, fmt.Sprintf("%d", mb.Size))
3949 status = append(status, A, "0")
3952 status = append(status, A, "NIL")
3953 case "HIGHESTMODSEQ":
3955 status = append(status, A, fmt.Sprintf("%d", mb.ModSeq.Client()))
3956 case "DELETED-STORAGE":
3958 // How much storage space could be reclaimed by expunging messages with the
3959 // \Deleted flag. We could keep track of this number and return it efficiently.
3960 // Calculating it each time can be slow, and we don't know if clients request it.
3961 // Clients are not likely to set the deleted flag without immediately expunging
3962 // nowadays. Let's wait for something to need it to go through the trouble, and
3963 // always return 0 for now.
3964 status = append(status, A, "0")
3966 xsyntaxErrorf("unknown attribute %q", a)
3969 return fmt.Sprintf("* STATUS %s (%s)", mailboxt(mb.Name).pack(c), strings.Join(status, " "))
3972func flaglist(fl store.Flags, keywords []string) listspace {
3974 flag := func(v bool, s string) {
3976 l = append(l, bare(s))
3979 flag(fl.Seen, `\Seen`)
3980 flag(fl.Answered, `\Answered`)
3981 flag(fl.Flagged, `\Flagged`)
3982 flag(fl.Deleted, `\Deleted`)
3983 flag(fl.Draft, `\Draft`)
3984 flag(fl.Forwarded, `$Forwarded`)
3985 flag(fl.Junk, `$Junk`)
3986 flag(fl.Notjunk, `$NotJunk`)
3987 flag(fl.Phishing, `$Phishing`)
3988 flag(fl.MDNSent, `$MDNSent`)
3989 for _, k := range keywords {
3990 l = append(l, bare(k))
3995// Append adds a message to a mailbox.
3996// The MULTIAPPEND extension is implemented, allowing multiple flags/datetime/data
3999// State: Authenticated and selected.
4000func (c *conn) cmdAppend(tag, cmd string, p *parser) {
4004 // A message that we've (partially) read from the client, and will be delivering to
4006 type appendMsg struct {
4007 storeFlags store.Flags
4011 file *os.File // Message file we are appending. Can be nil if we are writing to a nopWriteCloser due to being over quota.
4014 m store.Message // New message. Delivered file for m.ID is removed on error.
4017 var appends []*appendMsg
4020 for _, a := range appends {
4021 if !commit && a.m.ID != 0 {
4022 p := c.account.MessagePath(a.m.ID)
4024 c.xsanity(err, "cleaning up temporary append file after error")
4031 name := p.xmailbox()
4034 // Check how much quota space is available. We'll keep track of remaining quota as
4035 // we accept multiple messages.
4036 quotaMsgMax := c.account.QuotaMessageSize()
4037 quotaUnlimited := quotaMsgMax == 0
4038 var quotaAvail int64
4040 if !quotaUnlimited {
4041 c.account.WithRLock(func() {
4042 c.xdbread(func(tx *bstore.Tx) {
4043 du := store.DiskUsage{ID: 1}
4045 xcheckf(err, "get quota disk usage")
4046 quotaAvail = quotaMsgMax - du.MessageSize
4051 var overQuota bool // For response code.
4052 var cancel bool // In case we've seen zero-sized message append.
4055 // Append msg early, for potential cleanup.
4057 appends = append(appends, &a)
4059 if p.hasPrefix("(") {
4060 // Error must be a syntax error, to properly abort the connection due to literal.
4062 a.storeFlags, a.keywords, err = store.ParseFlagsKeywords(p.xflagList())
4064 xsyntaxErrorf("parsing flags: %v", err)
4068 if p.hasPrefix(`"`) {
4069 a.time = p.xdateTime()
4074 // todo: only with utf8 should we we accept message headers with utf-8. we currently always accept them.
4075 // todo: this is only relevant if we also support the CATENATE extension?
4077 utf8 := p.take("UTF8 (")
4082 // For utf8, we already consumed the required ~ above.
4083 size, synclit := p.xliteralSize(!utf8, false)
4085 if !quotaUnlimited && !overQuota {
4087 overQuota = quotaAvail < 0
4095 // Check for mailbox on first iteration.
4096 if len(appends) <= 1 {
4097 name = xcheckmailboxname(name, true)
4098 c.xdbread(func(tx *bstore.Tx) {
4099 c.xmailbox(tx, name, "TRYCREATE")
4105 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", quotaMsgMax)
4110 xuserErrorf("empty message, cancelling append")
4113 // Read the message into a temporary file.
4115 a.file, err = store.CreateMessageTemp(c.log, "imap-append")
4116 xcheckf(err, "creating temp file for message")
4117 defer store.CloseRemoveTempFile(c.log, a.file, "temporary message file")
4122 // We'll discard the message and return an error as soon as we can (possible
4123 // synchronizing literal of next message, or after we've seen all messages).
4124 if overQuota || cancel {
4128 a.file, err = store.CreateMessageTemp(c.log, "imap-append")
4129 xcheckf(err, "creating temp file for message")
4130 defer store.CloseRemoveTempFile(c.log, a.file, "temporary message file")
4135 defer c.xtracewrite(mlog.LevelTracedata)()
4136 a.mw = message.NewWriter(f)
4137 msize, err := io.Copy(a.mw, io.LimitReader(c.br, size))
4138 c.xtracewrite(mlog.LevelTrace) // Restore.
4140 // Cannot use xcheckf due to %w handling of errIO.
4141 c.xbrokenf("reading literal message: %s (%w)", err, errIO)
4144 c.xbrokenf("read %d bytes for message, expected %d (%w)", msize, size, errIO)
4148 line := c.xreadline(false)
4149 p = newParser(line, c)
4154 // The MULTIAPPEND extension allows more appends.
4161 name = xcheckmailboxname(name, true)
4165 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", quotaMsgMax)
4170 xuserErrorf("empty message, cancelling append")
4173 var mb store.Mailbox
4175 var pendingChanges []store.Change
4177 // In case of panic.
4178 c.flushChanges(pendingChanges)
4183 c.account.WithWLock(func() {
4184 var changes []store.Change
4186 c.xdbwrite(func(tx *bstore.Tx) {
4187 mb = c.xmailbox(tx, name, "TRYCREATE")
4189 nkeywords := len(mb.Keywords)
4191 // Check quota for all messages at once.
4192 ok, maxSize, err := c.account.CanAddMessageSize(tx, totalSize)
4193 xcheckf(err, "checking quota")
4196 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
4199 modseq, err := c.account.NextModSeq(tx)
4200 xcheckf(err, "get next mod seq")
4204 msgDirs := map[string]struct{}{}
4205 for _, a := range appends {
4206 a.m = store.Message{
4208 MailboxOrigID: mb.ID,
4210 Flags: a.storeFlags,
4211 Keywords: a.keywords,
4217 // todo: do a single junk training
4218 err = c.account.MessageAdd(c.log, tx, &mb, &a.m, a.file, store.AddOpts{SkipDirSync: true})
4219 xcheckf(err, "delivering message")
4221 changes = append(changes, a.m.ChangeAddUID(mb))
4223 msgDirs[filepath.Dir(c.account.MessagePath(a.m.ID))] = struct{}{}
4226 changes = append(changes, mb.ChangeCounts())
4227 if nkeywords != len(mb.Keywords) {
4228 changes = append(changes, mb.ChangeKeywords())
4231 err = tx.Update(&mb)
4232 xcheckf(err, "updating mailbox counts")
4234 for dir := range msgDirs {
4235 err := moxio.SyncDir(c.log, dir)
4236 xcheckf(err, "sync dir")
4242 // Fetch pending changes, possibly with new UIDs, so we can apply them before adding our own new UID.
4243 overflow, pendingChanges = c.comm.Get()
4245 // Broadcast the change to other connections.
4246 c.broadcast(changes)
4249 if c.mailboxID == mb.ID {
4251 pendingChanges = nil
4252 c.xapplyChanges(overflow, l, true)
4253 for _, a := range appends {
4254 c.uidAppend(a.m.UID)
4256 // todo spec: with condstore/qresync, is there a mechanism to let the client know the modseq for the appended uid? in theory an untagged fetch with the modseq after the OK APPENDUID could make sense, but this probably isn't allowed.
4257 c.xbwritelinef("* %d EXISTS", c.exists)
4263 if len(appends) == 1 {
4264 uidset = fmt.Sprintf("%d", appends[0].m.UID)
4266 uidset = fmt.Sprintf("%d:%d", appends[0].m.UID, appends[len(appends)-1].m.UID)
4268 c.xwriteresultf("%s OK [APPENDUID %d %s] appended", tag, mb.UIDValidity, uidset)
4271// Idle makes a client wait until the server sends untagged updates, e.g. about
4272// message delivery or mailbox create/rename/delete/subscription, etc. It allows a
4273// client to get updates in real-time, not needing the use for NOOP.
4275// State: Authenticated and selected.
4276func (c *conn) cmdIdle(tag, cmd string, p *parser) {
4283 c.xwritelinef("+ waiting")
4285 // With NOTIFY enabled, flush all pending changes.
4286 if c.notify != nil && len(c.notify.Delayed) > 0 {
4287 c.xapplyChanges(false, nil, true)
4295 case le := <-c.lineChan():
4297 if err := le.err; err != nil {
4298 if errors.Is(le.err, os.ErrDeadlineExceeded) {
4299 err := c.conn.SetDeadline(time.Now().Add(10 * time.Second))
4300 c.log.Check(err, "setting deadline")
4301 c.xwritelinef("* BYE inactive")
4304 if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) {
4305 c.xbrokenf("%s (%w)", err, errIO)
4311 case <-c.comm.Pending:
4312 overflow, changes := c.comm.Get()
4313 c.xapplyChanges(overflow, changes, true)
4315 case <-mox.Shutdown.Done():
4317 c.xwritelinef("* BYE shutting down")
4318 c.xbrokenf("shutting down (%w)", errIO)
4322 // Reset the write deadline. In case of little activity, with a command timeout of
4323 // 30 minutes, we have likely passed it.
4324 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
4325 c.log.Check(err, "setting write deadline")
4327 if strings.ToUpper(line) != "DONE" {
4328 // We just close the connection because our protocols are out of sync.
4329 c.xbrokenf("%w: in IDLE, expected DONE", errIO)
4335// Return the quota root for a mailbox name and any current quota's.
4337// State: Authenticated and selected.
4338func (c *conn) cmdGetquotaroot(tag, cmd string, p *parser) {
4343 name := p.xmailbox()
4346 // This mailbox does not have to exist. Caller just wants to know which limits
4347 // would apply. We only have one limit, so we don't use the name otherwise.
4349 name = xcheckmailboxname(name, true)
4351 // Get current usage for account.
4352 var quota, size int64 // Account only has a quota if > 0.
4353 c.account.WithRLock(func() {
4354 quota = c.account.QuotaMessageSize()
4356 c.xdbread(func(tx *bstore.Tx) {
4357 du := store.DiskUsage{ID: 1}
4359 xcheckf(err, "gather used quota")
4360 size = du.MessageSize
4365 // We only have one per account quota, we name it "" like the examples in the RFC.
4367 c.xbwritelinef(`* QUOTAROOT %s ""`, astring(name).pack(c))
4369 // We only write the quota response if there is a limit. The syntax doesn't allow
4370 // an empty list, so we cannot send the current disk usage if there is no limit.
4373 c.xbwritelinef(`* QUOTA "" (STORAGE %d %d)`, (size+1024-1)/1024, (quota+1024-1)/1024)
4378// Return the quota for a quota root.
4380// State: Authenticated and selected.
4381func (c *conn) cmdGetquota(tag, cmd string, p *parser) {
4386 root := p.xastring()
4389 // We only have a per-account root called "".
4391 xuserErrorf("unknown quota root")
4394 var quota, size int64
4395 c.account.WithRLock(func() {
4396 quota = c.account.QuotaMessageSize()
4398 c.xdbread(func(tx *bstore.Tx) {
4399 du := store.DiskUsage{ID: 1}
4401 xcheckf(err, "gather used quota")
4402 size = du.MessageSize
4407 // We only write the quota response if there is a limit. The syntax doesn't allow
4408 // an empty list, so we cannot send the current disk usage if there is no limit.
4411 c.xbwritelinef(`* QUOTA "" (STORAGE %d %d)`, (size+1024-1)/1024, (quota+1024-1)/1024)
4416// Check is an old deprecated command that is supposed to execute some mailbox consistency checks.
4419func (c *conn) cmdCheck(tag, cmd string, p *parser) {
4425 c.account.WithRLock(func() {
4426 c.xdbread(func(tx *bstore.Tx) {
4427 c.xmailboxID(tx, c.mailboxID) // Validate.
4434// Close undoes select/examine, closing the currently opened mailbox and deleting
4435// messages that were marked for deletion with the \Deleted flag.
4438func (c *conn) cmdClose(tag, cmd string, p *parser) {
4445 c.xexpunge(nil, true)
4451// expunge messages marked for deletion in currently selected/active mailbox.
4452// if uidSet is not nil, only messages matching the set are expunged.
4454// Messages that have been marked expunged from the database are returned. While
4455// other sessions still reference the message, it is not cleared from the database
4456// yet, and the message file is not yet removed.
4458// The highest modseq in the mailbox is returned, typically associated with the
4459// removal of the messages, but if no messages were expunged the current latest max
4460// modseq for the mailbox is returned.
4461func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (expunged []store.Message, highestModSeq store.ModSeq) {
4462 c.account.WithWLock(func() {
4463 var changes []store.Change
4465 c.xdbwrite(func(tx *bstore.Tx) {
4466 mb, err := store.MailboxID(tx, c.mailboxID)
4467 if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
4468 if missingMailboxOK {
4472 xusercodeErrorf("NONEXISTENT", "%w", store.ErrUnknownMailbox)
4474 xcheckf(err, "get mailbox")
4476 xlastUID := c.newCachedLastUID(tx, c.mailboxID, func(err error) { xuserErrorf("%s", err) })
4478 qm := bstore.QueryTx[store.Message](tx)
4479 qm.FilterNonzero(store.Message{MailboxID: c.mailboxID})
4480 qm.FilterEqual("Deleted", true)
4481 qm.FilterEqual("Expunged", false)
4482 qm.FilterLess("UID", c.uidnext)
4483 qm.FilterFn(func(m store.Message) bool {
4484 // Only remove if this session knows about the message and if present in optional
4486 return uidSet == nil || uidSet.xcontainsKnownUID(m.UID, c.searchResult, xlastUID)
4489 expunged, err = qm.List()
4490 xcheckf(err, "listing messages to expunge")
4492 if len(expunged) == 0 {
4493 highestModSeq = mb.ModSeq
4497 // Assign new modseq.
4498 modseq, err := c.account.NextModSeq(tx)
4499 xcheckf(err, "assigning next modseq")
4500 highestModSeq = modseq
4503 chremuids, chmbcounts, err := c.account.MessageRemove(c.log, tx, modseq, &mb, store.RemoveOpts{}, expunged...)
4504 xcheckf(err, "expunging messages")
4505 changes = append(changes, chremuids, chmbcounts)
4507 err = tx.Update(&mb)
4508 xcheckf(err, "update mailbox")
4511 c.broadcast(changes)
4514 return expunged, highestModSeq
4517// Unselect is similar to close in that it closes the currently active mailbox, but
4518// it does not remove messages marked for deletion.
4521func (c *conn) cmdUnselect(tag, cmd string, p *parser) {
4531// Expunge deletes messages marked with \Deleted in the currently selected mailbox.
4532// Clients are wiser to use UID EXPUNGE because it allows a UID sequence set to
4533// explicitly opt in to removing specific messages.
4536func (c *conn) cmdExpunge(tag, cmd string, p *parser) {
4543 xuserErrorf("mailbox open in read-only mode")
4546 c.cmdxExpunge(tag, cmd, nil)
4549// UID expunge deletes messages marked with \Deleted in the currently selected
4550// mailbox if they match a UID sequence set.
4553func (c *conn) cmdUIDExpunge(tag, cmd string, p *parser) {
4558 uidSet := p.xnumSet()
4562 xuserErrorf("mailbox open in read-only mode")
4565 c.cmdxExpunge(tag, cmd, &uidSet)
4568// Permanently delete messages for the currently selected/active mailbox. If uidset
4569// is not nil, only those UIDs are expunged.
4571func (c *conn) cmdxExpunge(tag, cmd string, uidSet *numSet) {
4574 expunged, highestModSeq := c.xexpunge(uidSet, false)
4577 var vanishedUIDs numSet
4578 qresync := c.enabled[capQresync]
4579 for _, m := range expunged {
4583 vanishedUIDs.append(uint32(m.UID))
4586 seq := c.xsequence(m.UID)
4587 c.sequenceRemove(seq, m.UID)
4589 vanishedUIDs.append(uint32(m.UID))
4591 c.xbwritelinef("* %d EXPUNGE", seq)
4594 if !vanishedUIDs.empty() {
4596 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
4597 c.xbwritelinef("* VANISHED %s", s)
4601 if c.enabled[capCondstore] {
4602 c.xwriteresultf("%s OK [HIGHESTMODSEQ %d] expunged", tag, highestModSeq.Client())
4609func (c *conn) cmdSearch(tag, cmd string, p *parser) {
4610 c.cmdxSearch(false, false, tag, cmd, p)
4614func (c *conn) cmdUIDSearch(tag, cmd string, p *parser) {
4615 c.cmdxSearch(true, false, tag, cmd, p)
4619func (c *conn) cmdFetch(tag, cmd string, p *parser) {
4620 c.cmdxFetch(false, tag, cmd, p)
4624func (c *conn) cmdUIDFetch(tag, cmd string, p *parser) {
4625 c.cmdxFetch(true, tag, cmd, p)
4629func (c *conn) cmdStore(tag, cmd string, p *parser) {
4630 c.cmdxStore(false, tag, cmd, p)
4634func (c *conn) cmdUIDStore(tag, cmd string, p *parser) {
4635 c.cmdxStore(true, tag, cmd, p)
4639func (c *conn) cmdCopy(tag, cmd string, p *parser) {
4640 c.cmdxCopy(false, tag, cmd, p)
4644func (c *conn) cmdUIDCopy(tag, cmd string, p *parser) {
4645 c.cmdxCopy(true, tag, cmd, p)
4649func (c *conn) cmdMove(tag, cmd string, p *parser) {
4650 c.cmdxMove(false, tag, cmd, p)
4654func (c *conn) cmdUIDMove(tag, cmd string, p *parser) {
4655 c.cmdxMove(true, tag, cmd, p)
4659func (c *conn) cmdReplace(tag, cmd string, p *parser) {
4660 c.cmdxReplace(false, tag, cmd, p)
4664func (c *conn) cmdUIDReplace(tag, cmd string, p *parser) {
4665 c.cmdxReplace(true, tag, cmd, p)
4668func (c *conn) gatherCopyMoveUIDs(tx *bstore.Tx, isUID bool, nums numSet) []store.UID {
4669 // Gather uids, then sort so we can return a consistently simple and hard to
4670 // misinterpret COPYUID/MOVEUID response. It seems safer to have UIDs in ascending
4671 // order, because requested uid set of 12:10 is equal to 10:12, so if we would just
4672 // echo whatever the client sends us without reordering, the client can reorder our
4673 // response and interpret it differently than we intended.
4675 return c.xnumSetEval(tx, isUID, nums)
4678// Copy copies messages from the currently selected/active mailbox to another named
4682func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
4689 name := p.xmailbox()
4692 name = xcheckmailboxname(name, true)
4694 // Files that were created during the copy. Remove them if the operation fails.
4697 for _, id := range newIDs {
4698 p := c.account.MessagePath(id)
4700 c.xsanity(err, "cleaning up created file")
4705 var uids []store.UID
4707 var mbDst store.Mailbox
4709 var newUIDs []store.UID
4710 var flags []store.Flags
4711 var keywords [][]string
4712 var modseq store.ModSeq // For messages in new mailbox, assigned when first message is copied.
4714 c.account.WithWLock(func() {
4716 c.xdbwrite(func(tx *bstore.Tx) {
4717 mbSrc := c.xmailboxID(tx, c.mailboxID) // Validate.
4719 mbDst = c.xmailbox(tx, name, "TRYCREATE")
4720 if mbDst.ID == mbSrc.ID {
4721 xuserErrorf("cannot copy to currently selected mailbox")
4724 uids = c.gatherCopyMoveUIDs(tx, isUID, nums)
4727 xuserErrorf("no matching messages to copy")
4730 nkeywords = len(mbDst.Keywords)
4733 modseq, err = c.account.NextModSeq(tx)
4734 xcheckf(err, "assigning next modseq")
4735 mbSrc.ModSeq = modseq
4736 mbDst.ModSeq = modseq
4738 err = tx.Update(&mbSrc)
4739 xcheckf(err, "updating source mailbox for modseq")
4741 // Reserve the uids in the destination mailbox.
4742 uidFirst := mbDst.UIDNext
4743 err = mbDst.UIDNextAdd(len(uids))
4744 xcheckf(err, "adding uid")
4746 // Fetch messages from database.
4747 q := bstore.QueryTx[store.Message](tx)
4748 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
4749 q.FilterEqual("UID", slicesAny(uids)...)
4750 q.FilterEqual("Expunged", false)
4751 xmsgs, err := q.List()
4752 xcheckf(err, "fetching messages")
4754 if len(xmsgs) != len(uids) {
4755 xserverErrorf("uid and message mismatch")
4758 // See if quota allows copy.
4760 for _, m := range xmsgs {
4763 if ok, maxSize, err := c.account.CanAddMessageSize(tx, totalSize); err != nil {
4764 xcheckf(err, "checking quota")
4767 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
4769 err = c.account.AddMessageSize(c.log, tx, totalSize)
4770 xcheckf(err, "updating disk usage")
4772 msgs := map[store.UID]store.Message{}
4773 for _, m := range xmsgs {
4776 nmsgs := make([]store.Message, len(xmsgs))
4778 conf, _ := c.account.Conf()
4780 mbKeywords := map[string]struct{}{}
4783 // Insert new messages into database.
4784 var origMsgIDs, newMsgIDs []int64
4785 for i, uid := range uids {
4788 xuserErrorf("messages changed, could not fetch requested uid")
4791 origMsgIDs = append(origMsgIDs, origID)
4793 m.UID = uidFirst + store.UID(i)
4794 m.CreateSeq = modseq
4796 m.MailboxID = mbDst.ID
4797 if m.IsReject && m.MailboxDestinedID != 0 {
4798 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
4799 // is used for reputation calculation during future deliveries.
4800 m.MailboxOrigID = m.MailboxDestinedID
4804 m.JunkFlagsForMailboxMove(mbSrc, mbDst, conf)
4806 err := tx.Insert(&m)
4807 xcheckf(err, "inserting message")
4810 newUIDs = append(newUIDs, m.UID)
4811 newMsgIDs = append(newMsgIDs, m.ID)
4812 flags = append(flags, m.Flags)
4813 keywords = append(keywords, m.Keywords)
4814 for _, kw := range m.Keywords {
4815 mbKeywords[kw] = struct{}{}
4818 qmr := bstore.QueryTx[store.Recipient](tx)
4819 qmr.FilterNonzero(store.Recipient{MessageID: origID})
4820 mrs, err := qmr.List()
4821 xcheckf(err, "listing message recipients")
4822 for _, mr := range mrs {
4825 err := tx.Insert(&mr)
4826 xcheckf(err, "inserting message recipient")
4829 mbDst.Add(m.MailboxCounts())
4832 mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, slices.Sorted(maps.Keys(mbKeywords)))
4834 err = tx.Update(&mbDst)
4835 xcheckf(err, "updating destination mailbox for uids, keywords and counts")
4837 // Copy message files to new message ID's.
4838 syncDirs := map[string]struct{}{}
4839 for i := range origMsgIDs {
4840 src := c.account.MessagePath(origMsgIDs[i])
4841 dst := c.account.MessagePath(newMsgIDs[i])
4842 dstdir := filepath.Dir(dst)
4843 if _, ok := syncDirs[dstdir]; !ok {
4844 os.MkdirAll(dstdir, 0770)
4845 syncDirs[dstdir] = struct{}{}
4847 err := moxio.LinkOrCopy(c.log, dst, src, nil, true)
4848 xcheckf(err, "link or copy file %q to %q", src, dst)
4849 newIDs = append(newIDs, newMsgIDs[i])
4852 for dir := range syncDirs {
4853 err := moxio.SyncDir(c.log, dir)
4854 xcheckf(err, "sync directory")
4857 err = c.account.RetrainMessages(context.TODO(), c.log, tx, nmsgs)
4858 xcheckf(err, "train copied messages")
4863 // Broadcast changes to other connections.
4864 if len(newUIDs) > 0 {
4865 changes := make([]store.Change, 0, len(newUIDs)+2)
4866 for i, uid := range newUIDs {
4867 add := store.ChangeAddUID{
4868 MailboxID: mbDst.ID,
4872 Keywords: keywords[i],
4873 MessageCountIMAP: mbDst.MessageCountIMAP(),
4874 Unseen: uint32(mbDst.MailboxCounts.Unseen),
4876 changes = append(changes, add)
4878 changes = append(changes, mbDst.ChangeCounts())
4879 if nkeywords != len(mbDst.Keywords) {
4880 changes = append(changes, mbDst.ChangeKeywords())
4882 c.broadcast(changes)
4887 c.xwriteresultf("%s OK [COPYUID %d %s %s] copied", tag, mbDst.UIDValidity, compactUIDSet(uids).String(), compactUIDSet(newUIDs).String())
4890// Move moves messages from the currently selected/active mailbox to a named mailbox.
4893func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) {
4900 name := p.xmailbox()
4903 name = xcheckmailboxname(name, true)
4906 xuserErrorf("mailbox open in read-only mode")
4910 var uids []store.UID
4912 var mbDst store.Mailbox
4913 var uidFirst store.UID
4914 var modseq store.ModSeq
4916 var cleanupIDs []int64
4918 for _, id := range cleanupIDs {
4919 p := c.account.MessagePath(id)
4921 c.xsanity(err, "removing destination message file %v", p)
4925 c.account.WithWLock(func() {
4926 var changes []store.Change
4928 c.xdbwrite(func(tx *bstore.Tx) {
4929 mbSrc := c.xmailboxID(tx, c.mailboxID) // Validate.
4930 mbDst = c.xmailbox(tx, name, "TRYCREATE")
4931 if mbDst.ID == c.mailboxID {
4932 xuserErrorf("cannot move to currently selected mailbox")
4935 uids = c.gatherCopyMoveUIDs(tx, isUID, nums)
4938 xuserErrorf("no matching messages to move")
4941 uidFirst = mbDst.UIDNext
4943 // Assign a new modseq, for the new records and for the expunged records.
4945 modseq, err = c.account.NextModSeq(tx)
4946 xcheckf(err, "assigning next modseq")
4948 // Make query selecting messages to move.
4949 q := bstore.QueryTx[store.Message](tx)
4950 q.FilterNonzero(store.Message{MailboxID: mbSrc.ID})
4951 q.FilterEqual("UID", slicesAny(uids)...)
4952 q.FilterEqual("Expunged", false)
4955 newIDs, chl := c.xmoveMessages(tx, q, len(uids), modseq, &mbSrc, &mbDst)
4956 changes = append(changes, chl...)
4962 c.broadcast(changes)
4967 newUIDs := numSet{ranges: []numRange{{setNumber{number: uint32(uidFirst)}, &setNumber{number: uint32(mbDst.UIDNext - 1)}}}}
4968 c.xbwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), newUIDs.String())
4969 qresync := c.enabled[capQresync]
4970 var vanishedUIDs numSet
4971 for i := range uids {
4975 vanishedUIDs.append(uint32(uids[i]))
4979 seq := c.xsequence(uids[i])
4980 c.sequenceRemove(seq, uids[i])
4982 vanishedUIDs.append(uint32(uids[i]))
4984 c.xbwritelinef("* %d EXPUNGE", seq)
4987 if !vanishedUIDs.empty() {
4989 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
4990 c.xbwritelinef("* VANISHED %s", s)
4996 c.xwriteresultf("%s OK [HIGHESTMODSEQ %d] move", tag, modseq.Client())
5002// q must yield messages from a single mailbox.
5003func (c *conn) xmoveMessages(tx *bstore.Tx, q *bstore.Query[store.Message], expectCount int, modseq store.ModSeq, mbSrc, mbDst *store.Mailbox) (newIDs []int64, changes []store.Change) {
5004 newIDs = make([]int64, 0, expectCount)
5010 for _, id := range newIDs {
5011 p := c.account.MessagePath(id)
5013 c.xsanity(err, "removing added message file %v", p)
5018 mbSrc.ModSeq = modseq
5019 mbDst.ModSeq = modseq
5024 err := jf.CloseDiscard()
5025 c.log.Check(err, "closing junk filter after error")
5029 accConf, _ := c.account.Conf()
5031 changeRemoveUIDs := store.ChangeRemoveUIDs{
5032 MailboxID: mbSrc.ID,
5035 changes = make([]store.Change, 0, expectCount+4) // mbsrc removeuids, mbsrc counts, mbdst counts, mbdst keywords
5037 nkeywords := len(mbDst.Keywords)
5041 xcheckf(err, "listing messages to move")
5043 if expectCount > 0 && len(l) != expectCount {
5044 xcheckf(fmt.Errorf("moved %d messages, expected %d", len(l), expectCount), "move messages")
5047 // For newly created message directories that we sync after hardlinking/copying files.
5048 syncDirs := map[string]struct{}{}
5050 for _, om := range l {
5052 nm.MailboxID = mbDst.ID
5053 nm.UID = mbDst.UIDNext
5054 err := mbDst.UIDNextAdd(1)
5055 xcheckf(err, "adding uid")
5057 nm.CreateSeq = modseq
5059 if nm.IsReject && nm.MailboxDestinedID != 0 {
5060 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
5061 // is used for reputation calculation during future deliveries.
5062 nm.MailboxOrigID = nm.MailboxDestinedID
5067 nm.JunkFlagsForMailboxMove(*mbSrc, *mbDst, accConf)
5069 err = tx.Update(&nm)
5070 xcheckf(err, "updating message with new mailbox")
5072 mbDst.Add(nm.MailboxCounts())
5074 mbSrc.Sub(om.MailboxCounts())
5078 om.TrainedJunk = nil
5079 err = tx.Insert(&om)
5080 xcheckf(err, "inserting expunged message in old mailbox")
5082 dstPath := c.account.MessagePath(om.ID)
5083 dstDir := filepath.Dir(dstPath)
5084 if _, ok := syncDirs[dstDir]; !ok {
5085 os.MkdirAll(dstDir, 0770)
5086 syncDirs[dstDir] = struct{}{}
5089 err = moxio.LinkOrCopy(c.log, dstPath, c.account.MessagePath(nm.ID), nil, false)
5090 xcheckf(err, "duplicating message in old mailbox for current sessions")
5091 newIDs = append(newIDs, nm.ID)
5092 // We don't sync the directory. In case of a crash and files disappearing, the
5093 // eraser will simply not find the file at next startup.
5095 err = tx.Insert(&store.MessageErase{ID: om.ID, SkipUpdateDiskUsage: true})
5096 xcheckf(err, "insert message erase")
5098 mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, nm.Keywords)
5100 if accConf.JunkFilter != nil && nm.NeedsTraining() {
5101 // Lazily open junk filter.
5103 jf, _, err = c.account.OpenJunkFilter(context.TODO(), c.log)
5104 xcheckf(err, "open junk filter")
5106 err := c.account.RetrainMessage(context.TODO(), c.log, tx, jf, &nm)
5107 xcheckf(err, "retrain message after moving")
5110 changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID)
5111 changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID)
5112 changes = append(changes, nm.ChangeAddUID(*mbDst))
5114 xcheckf(err, "move messages")
5116 for dir := range syncDirs {
5117 err := moxio.SyncDir(c.log, dir)
5118 xcheckf(err, "sync directory")
5121 changeRemoveUIDs.UIDNext = mbDst.UIDNext
5122 changeRemoveUIDs.MessageCountIMAP = mbDst.MessageCountIMAP()
5123 changeRemoveUIDs.Unseen = uint32(mbDst.MailboxCounts.Unseen)
5124 changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts())
5126 err = tx.Update(mbSrc)
5127 xcheckf(err, "updating counts for inbox")
5129 changes = append(changes, mbDst.ChangeCounts())
5130 if len(mbDst.Keywords) > nkeywords {
5131 changes = append(changes, mbDst.ChangeKeywords())
5134 err = tx.Update(mbDst)
5135 xcheckf(err, "updating uidnext and counts in destination mailbox")
5140 xcheckf(err, "saving junk filter")
5147// Store sets a full set of flags, or adds/removes specific flags.
5150func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) {
5157 var unchangedSince *int64
5160 p.xtake("UNCHANGEDSINCE")
5167 c.xensureCondstore(nil)
5169 var plus, minus bool
5172 } else if p.take("-") {
5176 silent := p.take(".SILENT")
5178 var flagstrs []string
5179 if p.hasPrefix("(") {
5180 flagstrs = p.xflagList()
5182 flagstrs = append(flagstrs, p.xflag())
5184 flagstrs = append(flagstrs, p.xflag())
5190 xuserErrorf("mailbox open in read-only mode")
5193 flags, keywords, err := store.ParseFlagsKeywords(flagstrs)
5195 xuserErrorf("parsing flags: %v", err)
5197 var mask store.Flags
5199 mask, flags = flags, store.FlagsAll
5201 mask, flags = flags, store.Flags{}
5203 mask = store.FlagsAll
5206 var mb, origmb store.Mailbox
5207 var updated []store.Message
5208 var changed []store.Message // ModSeq more recent than unchangedSince, will be in MODIFIED response code, and we will send untagged fetch responses so client is up to date.
5209 var modseq store.ModSeq // Assigned when needed.
5210 modified := map[int64]bool{}
5212 c.account.WithWLock(func() {
5213 var mbKwChanged bool
5214 var changes []store.Change
5216 c.xdbwrite(func(tx *bstore.Tx) {
5217 mb = c.xmailboxID(tx, c.mailboxID) // Validate.
5220 uids := c.xnumSetEval(tx, isUID, nums)
5226 // Ensure keywords are in mailbox.
5228 mb.Keywords, mbKwChanged = store.MergeKeywords(mb.Keywords, keywords)
5230 err := tx.Update(&mb)
5231 xcheckf(err, "updating mailbox with keywords")
5235 q := bstore.QueryTx[store.Message](tx)
5236 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
5237 q.FilterEqual("UID", slicesAny(uids)...)
5238 q.FilterEqual("Expunged", false)
5239 err := q.ForEach(func(m store.Message) error {
5240 // Client may specify a message multiple times, but we only process it once.
../rfc/7162:823
5245 mc := m.MailboxCounts()
5247 origFlags := m.Flags
5248 m.Flags = m.Flags.Set(mask, flags)
5249 oldKeywords := slices.Clone(m.Keywords)
5251 m.Keywords, _ = store.RemoveKeywords(m.Keywords, keywords)
5253 m.Keywords, _ = store.MergeKeywords(m.Keywords, keywords)
5255 m.Keywords = keywords
5258 keywordsChanged := func() bool {
5259 sort.Strings(oldKeywords)
5260 n := slices.Clone(m.Keywords)
5262 return !slices.Equal(oldKeywords, n)
5265 // If the message has a more recent modseq than the check requires, we won't modify
5266 // it and report in the final command response.
5269 // unchangedSince 0 always fails the check, we don't turn it into 1 like with our
5270 // internal modseqs. RFC implies that is not required for non-system flags, but we
5272 if unchangedSince != nil && m.ModSeq.Client() > *unchangedSince {
5273 changed = append(changed, m)
5278 // It requires that we keep track of the flags we think the client knows (but only
5279 // on this connection). We don't track that. It also isn't clear why this is
5280 // allowed because it is skipping the condstore conditional check, and the new
5281 // combination of flags could be unintended.
5284 if origFlags == m.Flags && !keywordsChanged() {
5285 // Note: since we didn't update the modseq, we are not adding m.ID to "modified",
5286 // it would skip the modseq check above. We still add m to list of updated, so we
5287 // send an untagged fetch response. But we don't broadcast it.
5288 updated = append(updated, m)
5293 mb.Add(m.MailboxCounts())
5295 // Assign new modseq for first actual change.
5298 modseq, err = c.account.NextModSeq(tx)
5299 xcheckf(err, "next modseq")
5303 modified[m.ID] = true
5304 updated = append(updated, m)
5306 changes = append(changes, m.ChangeFlags(origFlags, mb))
5308 return tx.Update(&m)
5310 xcheckf(err, "storing flags in messages")
5312 if mb.MailboxCounts != origmb.MailboxCounts || modseq != 0 {
5313 err := tx.Update(&mb)
5314 xcheckf(err, "updating mailbox counts")
5316 if mb.MailboxCounts != origmb.MailboxCounts {
5317 changes = append(changes, mb.ChangeCounts())
5320 changes = append(changes, mb.ChangeKeywords())
5323 err = c.account.RetrainMessages(context.TODO(), c.log, tx, updated)
5324 xcheckf(err, "training messages")
5327 c.broadcast(changes)
5330 // In the RFC, the section about STORE/UID STORE says we must return MODSEQ when
5331 // UNCHANGEDSINCE was specified. It does not specify it in case UNCHANGEDSINCE
5332 // isn't specified. For that case it does say MODSEQ is needed in unsolicited
5333 // untagged fetch responses. Implying that solicited untagged fetch responses
5334 // should not include MODSEQ (why else mention unsolicited explicitly?). But, in
5335 // the introduction to CONDSTORE it does explicitly specify MODSEQ should be
5336 // included in untagged fetch responses at all times with CONDSTORE-enabled
5337 // connections. It would have been better if the command behaviour was specified in
5338 // the command section, not the introduction to the extension.
5341 if !silent || c.enabled[capCondstore] {
5342 for _, m := range updated {
5345 args = append(args, fmt.Sprintf("FLAGS %s", flaglist(m.Flags, m.Keywords).pack(c)))
5347 if c.enabled[capCondstore] {
5348 args = append(args, fmt.Sprintf("MODSEQ (%d)", m.ModSeq.Client()))
5353 // Ensure list is non-empty.
5355 args = append(args, fmt.Sprintf("UID %d", m.UID))
5357 c.xbwritelinef("* %d UIDFETCH (%s)", m.UID, strings.Join(args, " "))
5359 args = append([]string{fmt.Sprintf("UID %d", m.UID)}, args...)
5360 c.xbwritelinef("* %d FETCH (%s)", c.xsequence(m.UID), strings.Join(args, " "))
5365 // We don't explicitly send flags for failed updated with silent set. The regular
5366 // notification will get the flags to the client.
5369 if len(changed) == 0 {
5374 // Write unsolicited untagged fetch responses for messages that didn't pass the
5377 var mnums []store.UID
5378 for _, m := range changed {
5381 c.xbwritelinef("* %d UIDFETCH (FLAGS %s MODSEQ (%d))", m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
5383 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s MODSEQ (%d))", c.xsequence(m.UID), m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
5386 mnums = append(mnums, m.UID)
5388 mnums = append(mnums, store.UID(c.xsequence(m.UID)))
5393 set := compactUIDSet(mnums)
5395 c.xwriteresultf("%s OK [MODIFIED %s] conditional store did not modify all", tag, set.String())