1// Package imapserver implements an IMAPv4 server, rev2 (RFC 9051) and rev1 with extensions (RFC 3501 and more).
7IMAP4rev2 includes functionality that was in extensions for IMAP4rev1. The
8extensions sometimes include features not in IMAP4rev2. We want IMAP4rev1-only
9implementations to use extensions, so we implement the full feature set of the
10extension and announce it as capability. The extensions: LITERAL+, IDLE,
11NAMESPACE, BINARY, UNSELECT, UIDPLUS, ESEARCH, SEARCHRES, SASL-IR, ENABLE,
12LIST-EXTENDED, SPECIAL-USE, MOVE, UTF8=ONLY.
14We take a liberty with UTF8=ONLY. We are supposed to wait for ENABLE of
15UTF8=ACCEPT or IMAP4rev2 before we respond with quoted strings that contain
16non-ASCII UTF-8. Until that's enabled, we do use UTF-7 for mailbox names. See
19- We never execute multiple commands at the same time for a connection. We expect a client to open multiple connections instead.
../rfc/9051:1110
20- Do not write output on a connection with an account lock held. Writing can block, a slow client could block account operations.
21- When handling commands that modify the selected mailbox, always check that the mailbox is not opened readonly. And always revalidate the selected mailbox, another session may have deleted the mailbox.
22- After making changes to an account/mailbox/message, you must broadcast changes. You must do this with the account lock held. Otherwise, other later changes (e.g. message deliveries) may be made and broadcast before changes that were made earlier. Make sure to commit changes in the database first, because the commit may fail.
23- Mailbox hierarchies are slash separated, no leading slash. We keep the case, except INBOX is renamed to Inbox, also for submailboxes in INBOX. We don't allow existence of a child where its parent does not exist. We have no \NoInferiors or \NoSelect. Newly created mailboxes are automatically subscribed.
24- For CONDSTORE and QRESYNC support, we set "modseq" for each change/expunge. Once expunged, a modseq doesn't change anymore. We don't yet remove old expunged records. The records aren't too big. Next step may be to let an admin reclaim space manually.
28- todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64?
29- todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command.
62 "golang.org/x/text/unicode/norm"
64 "github.com/prometheus/client_golang/prometheus"
65 "github.com/prometheus/client_golang/prometheus/promauto"
67 "github.com/mjl-/bstore"
68 "github.com/mjl-/flate"
70 "github.com/mjl-/mox/config"
71 "github.com/mjl-/mox/junk"
72 "github.com/mjl-/mox/message"
73 "github.com/mjl-/mox/metrics"
74 "github.com/mjl-/mox/mlog"
75 "github.com/mjl-/mox/mox-"
76 "github.com/mjl-/mox/moxio"
77 "github.com/mjl-/mox/moxvar"
78 "github.com/mjl-/mox/ratelimit"
79 "github.com/mjl-/mox/scram"
80 "github.com/mjl-/mox/store"
84 metricIMAPConnection = promauto.NewCounterVec(
85 prometheus.CounterOpts{
86 Name: "mox_imap_connection_total",
87 Help: "Incoming IMAP connections.",
90 "service", // imap, imaps
93 metricIMAPCommands = promauto.NewHistogramVec(
94 prometheus.HistogramOpts{
95 Name: "mox_imap_command_duration_seconds",
96 Help: "IMAP command duration and result codes in seconds.",
97 Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
101 "result", // ok, panic, ioerror, badsyntax, servererror, usererror, error
106var unhandledPanics atomic.Int64 // For tests.
108var limiterConnectionrate, limiterConnections *ratelimit.Limiter
111 // Also called by tests, so they don't trigger the rate limiter.
117 limiterConnectionrate = &ratelimit.Limiter{
118 WindowLimits: []ratelimit.WindowLimit{
121 Limits: [...]int64{300, 900, 2700},
125 limiterConnections = &ratelimit.Limiter{
126 WindowLimits: []ratelimit.WindowLimit{
128 Window: time.Duration(math.MaxInt64), // All of time.
129 Limits: [...]int64{30, 90, 270},
135// Delay after bad/suspicious behaviour. Tests set these to zero.
136var badClientDelay = time.Second // Before reads and after 1-byte writes for probably spammers.
137var authFailDelay = time.Second // After authentication failure.
139// Capabilities (extensions) the server supports. Connections will add a few more,
140// e.g. STARTTLS, LOGINDISABLED, AUTH=PLAIN.
142// We always announce support for SCRAM PLUS-variants, also on connections without
143// TLS. The client should not be selecting PLUS variants on non-TLS connections,
144// instead opting to do the bare SCRAM variant without indicating the server claims
145// to support the PLUS variant (skipping the server downgrade detection check).
146var serverCapabilities = strings.Join([]string{
162 "CREATE-SPECIAL-USE", //
165 "AUTH=SCRAM-SHA-256", //
167 "AUTH=SCRAM-SHA-1", //
170 "APPENDLIMIT=9223372036854775807", //
../rfc/7889:129, we support the max possible size, 1<<63 - 1
175 "QUOTA=RES-STORAGE", //
188 // "COMPRESS=DEFLATE", //
../rfc/4978, disabled for interoperability issues: The flate reader (inflate) still blocks on partial flushes, preventing progress.
195 connBroken bool // Once broken, we won't flush any more data.
196 tls bool // Whether TLS has been initialized.
197 viaHTTPS bool // Whether this connection came in via HTTPS (using TLS ALPN).
198 br *bufio.Reader // From remote, with TLS unwrapped in case of TLS, and possibly wrapping inflate.
199 tr *moxio.TraceReader // Kept to change trace level when reading/writing cmd/auth/data.
200 line chan lineErr // If set, instead of reading from br, a line is read from this channel. For reading a line in IDLE while also waiting for mailbox/account updates.
201 lastLine string // For detecting if syntax error is fatal, i.e. if this ends with a literal. Without crlf.
202 xbw *bufio.Writer // To remote, with TLS added in case of TLS, and possibly wrapping deflate, see conn.xflateWriter. Writes go through xtw to conn.Write, which panics on errors, hence the "x".
203 xtw *moxio.TraceWriter
204 xflateWriter *moxio.FlateWriter // For flushing output after flushing conn.xbw, and for closing.
205 xflateBW *bufio.Writer // Wraps raw connection writes, xflateWriter writes here, also needs flushing.
206 slow bool // If set, reads are done with a 1 second sleep, and writes are done 1 byte at a time, to keep spammers busy.
207 lastlog time.Time // For printing time since previous log line.
208 baseTLSConfig *tls.Config // Base TLS config to use for handshake.
210 noRequireSTARTTLS bool
211 cmd string // Currently executing, for deciding to xapplyChanges and logging.
212 cmdMetric string // Currently executing, for metrics.
214 ncmds int // Number of commands processed. Used to abort connection when first incoming command is unknown/invalid.
215 log mlog.Log // Used for all synchronous logging on this connection, see logbg for logging in a separate goroutine.
216 enabled map[capability]bool // All upper-case.
217 compress bool // Whether compression is enabled, via compress command.
218 notify *notify // For the NOTIFY extension. Event/change filtering active if non-nil.
220 // Set by SEARCH with SAVE. Can be used by commands accepting a sequence-set with
221 // value "$". When used, UIDs must be verified to still exist, because they may
222 // have been expunged. Cleared by a SELECT or EXAMINE.
223 // Nil means no searchResult is present. An empty list is a valid searchResult,
224 // just not matching any messages.
226 searchResult []store.UID
228 // userAgent is set by the ID command, which can happen at any time (before or
229 // after the authentication attempt we want to log it with).
231 // loginAttempt is set during authentication, typically picked up by the ID command
232 // that soon follows, or it will be flushed within 1s, or on connection teardown.
233 loginAttempt *store.LoginAttempt
234 loginAttemptTime time.Time
236 // Only set when connection has been authenticated. These can be set even when
237 // c.state is stateNotAuthenticated, for TLS client certificate authentication. In
238 // that case, credentials aren't used until the authentication command with the
239 // SASL "EXTERNAL" mechanism.
240 authFailed int // Number of failed auth attempts. For slowing down remote with many failures.
241 noPreauth bool // If set, don't switch connection to "authenticated" after TLS handshake with client certificate authentication.
242 username string // Full username as used during login.
243 account *store.Account
244 comm *store.Comm // For sending/receiving changes on mailboxes in account, e.g. from messages incoming on smtp, or another imap client.
246 mailboxID int64 // Only for StateSelected.
247 readonly bool // If opened mailbox is readonly.
248 uidonly bool // If uidonly is enabled, uids is empty and cannot be used.
249 uidnext store.UID // We don't return search/fetch/etc results for uids >= uidnext, which is updated when applying changes.
250 exists uint32 // Needed for uidonly, equal to len(uids) for non-uidonly sessions.
251 uids []store.UID // UIDs known in this session, sorted. todo future: store more space-efficiently, as ranges.
254// capability for use with ENABLED and CAPABILITY. We always keep this upper case,
255// e.g. IMAP4REV2. These values are treated case-insensitive, but it's easier for
256// comparison to just always have the same case.
257type capability string
260 capIMAP4rev2 capability = "IMAP4REV2"
261 capUTF8Accept capability = "UTF8=ACCEPT"
262 capCondstore capability = "CONDSTORE"
263 capQresync capability = "QRESYNC"
264 capMetadata capability = "METADATA"
265 capUIDOnly capability = "UIDONLY"
276 stateNotAuthenticated state = iota
281func stateCommands(cmds ...string) map[string]struct{} {
282 r := map[string]struct{}{}
283 for _, cmd := range cmds {
290 commandsStateAny = stateCommands("capability", "noop", "logout", "id")
291 commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login")
292 commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata", "compress", "esearch", "notify")
293 commandsStateSelected = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move", "replace", "uid replace", "esearch")
296// Commands that use sequence numbers. Cannot be used when UIDONLY is enabled.
297// Commands like UID SEARCH have additional checks for some parameters.
298var commandsSequence = stateCommands("search", "fetch", "store", "copy", "move", "replace")
300var commands = map[string]func(c *conn, tag, cmd string, p *parser){
302 "capability": (*conn).cmdCapability,
303 "noop": (*conn).cmdNoop,
304 "logout": (*conn).cmdLogout,
308 "starttls": (*conn).cmdStarttls,
309 "authenticate": (*conn).cmdAuthenticate,
310 "login": (*conn).cmdLogin,
312 // Authenticated and selected.
313 "enable": (*conn).cmdEnable,
314 "select": (*conn).cmdSelect,
315 "examine": (*conn).cmdExamine,
316 "create": (*conn).cmdCreate,
317 "delete": (*conn).cmdDelete,
318 "rename": (*conn).cmdRename,
319 "subscribe": (*conn).cmdSubscribe,
320 "unsubscribe": (*conn).cmdUnsubscribe,
321 "list": (*conn).cmdList,
322 "lsub": (*conn).cmdLsub,
323 "namespace": (*conn).cmdNamespace,
324 "status": (*conn).cmdStatus,
325 "append": (*conn).cmdAppend,
326 "idle": (*conn).cmdIdle,
327 "getquotaroot": (*conn).cmdGetquotaroot,
328 "getquota": (*conn).cmdGetquota,
329 "getmetadata": (*conn).cmdGetmetadata,
330 "setmetadata": (*conn).cmdSetmetadata,
331 "compress": (*conn).cmdCompress,
332 "esearch": (*conn).cmdEsearch,
336 "check": (*conn).cmdCheck,
337 "close": (*conn).cmdClose,
338 "unselect": (*conn).cmdUnselect,
339 "expunge": (*conn).cmdExpunge,
340 "uid expunge": (*conn).cmdUIDExpunge,
341 "search": (*conn).cmdSearch,
342 "uid search": (*conn).cmdUIDSearch,
343 "fetch": (*conn).cmdFetch,
344 "uid fetch": (*conn).cmdUIDFetch,
345 "store": (*conn).cmdStore,
346 "uid store": (*conn).cmdUIDStore,
347 "copy": (*conn).cmdCopy,
348 "uid copy": (*conn).cmdUIDCopy,
349 "move": (*conn).cmdMove,
350 "uid move": (*conn).cmdUIDMove,
352 "replace": (*conn).cmdReplace,
353 "uid replace": (*conn).cmdUIDReplace,
356var errIO = errors.New("io error") // For read/write errors and errors that should close the connection.
357var errProtocol = errors.New("protocol error") // For protocol errors for which a stack trace should be printed.
361// check err for sanity.
362// if not nil and checkSanity true (set during tests), then panic. if not nil during normal operation, just log.
363func (c *conn) xsanity(err error, format string, args ...any) {
368 panic(fmt.Errorf("%s: %s", fmt.Sprintf(format, args...), err))
370 c.log.Errorx(fmt.Sprintf(format, args...), err)
373func (c *conn) xbrokenf(format string, args ...any) {
375 panic(fmt.Errorf(format, args...))
380// Listen initializes all imap listeners for the configuration, and stores them for Serve to start them.
382 names := slices.Sorted(maps.Keys(mox.Conf.Static.Listeners))
383 for _, name := range names {
384 listener := mox.Conf.Static.Listeners[name]
386 var tlsConfig *tls.Config
387 if listener.TLS != nil {
388 tlsConfig = listener.TLS.Config
391 if listener.IMAP.Enabled {
392 port := config.Port(listener.IMAP.Port, 143)
393 for _, ip := range listener.IPs {
394 listen1("imap", name, ip, port, tlsConfig, false, listener.IMAP.NoRequireSTARTTLS)
398 if listener.IMAPS.Enabled {
399 port := config.Port(listener.IMAPS.Port, 993)
400 for _, ip := range listener.IPs {
401 listen1("imaps", name, ip, port, tlsConfig, true, false)
409func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config, xtls, noRequireSTARTTLS bool) {
410 log := mlog.New("imapserver", nil)
411 addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
412 if os.Getuid() == 0 {
413 log.Print("listening for imap",
414 slog.String("listener", listenerName),
415 slog.String("addr", addr),
416 slog.String("protocol", protocol))
418 network := mox.Network(ip)
419 ln, err := mox.Listen(network, addr)
421 log.Fatalx("imap: listen for imap", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
424 // Each listener gets its own copy of the config, so session keys between different
425 // ports on same listener aren't shared. We rotate session keys explicitly in this
426 // base TLS config because each connection clones the TLS config before using. The
427 // base TLS config would never get automatically managed/rotated session keys.
428 if tlsConfig != nil {
429 tlsConfig = tlsConfig.Clone()
430 mox.StartTLSSessionTicketKeyRefresher(mox.Shutdown, log, tlsConfig)
435 conn, err := ln.Accept()
437 log.Infox("imap: accept", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
441 metricIMAPConnection.WithLabelValues(protocol).Inc()
442 go serve(listenerName, mox.Cid(), tlsConfig, conn, xtls, noRequireSTARTTLS, false, "")
446 servers = append(servers, serve)
449// ServeTLSConn serves IMAP on a TLS connection.
450func ServeTLSConn(listenerName string, conn *tls.Conn, tlsConfig *tls.Config) {
451 serve(listenerName, mox.Cid(), tlsConfig, conn, true, false, true, "")
454func ServeConnPreauth(listenerName string, cid int64, conn net.Conn, preauthAddress string) {
455 serve(listenerName, cid, nil, conn, false, true, false, preauthAddress)
458// Serve starts serving on all listeners, launching a goroutine per listener.
460 for _, serve := range servers {
466// Logbg returns a logger for logging in the background (in a goroutine), eg for
467// logging LoginAttempts. The regular c.log has a handler that evaluates fields on
468// the connection at time of logging, which may happen at the same time as
469// modifications to those fields.
470func (c *conn) logbg() mlog.Log {
471 log := mlog.New("imapserver", nil).WithCid(c.cid)
472 if c.username != "" {
473 log = log.With(slog.String("username", c.username))
478// returns whether this connection accepts utf-8 in strings.
479func (c *conn) utf8strings() bool {
480 return c.enabled[capIMAP4rev2] || c.enabled[capUTF8Accept]
483func (c *conn) xdbwrite(fn func(tx *bstore.Tx)) {
484 err := c.account.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
488 xcheckf(err, "transaction")
491func (c *conn) xdbread(fn func(tx *bstore.Tx)) {
492 err := c.account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
496 xcheckf(err, "transaction")
499// Closes the currently selected/active mailbox, setting state from selected to authenticated.
500// Does not remove messages marked for deletion.
501func (c *conn) unselect() {
502 // Flush any pending delayed changes as if the mailbox is still selected. Probably
503 // better than causing STATUS responses for the mailbox being unselected but which
504 // is still selected.
505 c.flushNotifyDelayed()
507 if c.state == stateSelected {
508 c.state = stateAuthenticated
516func (c *conn) flushNotifyDelayed() {
520 delayed := c.notify.Delayed
521 c.notify.Delayed = nil
522 c.flushChanges(delayed)
525// flushChanges is called for NOTIFY changes we shouldn't send untagged messages
526// about but must process for message removals. We don't update the selected
527// mailbox message sequence numbers, since the client would have no idea we
528// adjusted message sequence numbers. Combined with NOTIFY NONE, this means
529// messages may be erased that the client thinks still exists in its session.
530func (c *conn) flushChanges(changes []store.Change) {
531 for _, change := range changes {
532 switch ch := change.(type) {
533 case store.ChangeRemoveUIDs:
534 c.comm.RemovalSeen(ch)
539func (c *conn) setSlow(on bool) {
541 c.log.Debug("connection changed to slow")
542 } else if !on && c.slow {
543 c.log.Debug("connection restored to regular pace")
548// Write makes a connection an io.Writer. It panics for i/o errors. These errors
549// are handled in the connection command loop.
550func (c *conn) Write(buf []byte) (int, error) {
558 err := c.conn.SetWriteDeadline(time.Now().Add(30 * time.Second))
559 c.log.Check(err, "setting write deadline")
561 nn, err := c.conn.Write(buf[:chunk])
563 c.xbrokenf("write: %s (%w)", err, errIO)
567 if len(buf) > 0 && badClientDelay > 0 {
568 mox.Sleep(mox.Context, badClientDelay)
574func (c *conn) xtraceread(level slog.Level) func() {
577 c.tr.SetTrace(mlog.LevelTrace)
581func (c *conn) xtracewrite(level slog.Level) func() {
583 c.xtw.SetTrace(level)
586 c.xtw.SetTrace(mlog.LevelTrace)
590// Cache of line buffers for reading commands.
592var bufpool = moxio.NewBufpool(8, 16*1024)
594// read line from connection, not going through line channel.
595func (c *conn) readline0() (string, error) {
596 if c.slow && badClientDelay > 0 {
597 mox.Sleep(mox.Context, badClientDelay)
600 d := 30 * time.Minute
601 if c.state == stateNotAuthenticated {
604 err := c.conn.SetReadDeadline(time.Now().Add(d))
605 c.log.Check(err, "setting read deadline")
607 line, err := bufpool.Readline(c.log, c.br)
608 if err != nil && errors.Is(err, moxio.ErrLineTooLong) {
609 return "", fmt.Errorf("%s (%w)", err, errProtocol)
610 } else if err != nil {
611 return "", fmt.Errorf("%s (%w)", err, errIO)
616func (c *conn) lineChan() chan lineErr {
618 c.line = make(chan lineErr, 1)
620 line, err := c.readline0()
621 c.line <- lineErr{line, err}
627// readline from either the c.line channel, or otherwise read from connection.
628func (c *conn) xreadline(readCmd bool) string {
634 line, err = le.line, le.err
636 line, err = c.readline0()
639 if readCmd && errors.Is(err, os.ErrDeadlineExceeded) {
640 err := c.conn.SetDeadline(time.Now().Add(10 * time.Second))
641 c.log.Check(err, "setting deadline")
642 c.xwritelinef("* BYE inactive")
645 if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) {
646 c.xbrokenf("%s (%w)", err, errIO)
652 // We typically respond immediately (IDLE is an exception).
653 // The client may not be reading, or may have disappeared.
654 // Don't wait more than 5 minutes before closing down the connection.
655 // The write deadline is managed in IDLE as well.
656 // For unauthenticated connections, we require the client to read faster.
657 wd := 5 * time.Minute
658 if c.state == stateNotAuthenticated {
659 wd = 30 * time.Second
661 err = c.conn.SetWriteDeadline(time.Now().Add(wd))
662 c.log.Check(err, "setting write deadline")
667// write tagged command response, but first write pending changes.
668func (c *conn) xwriteresultf(format string, args ...any) {
669 c.xbwriteresultf(format, args...)
673// write buffered tagged command response, but first write pending changes.
674func (c *conn) xbwriteresultf(format string, args ...any) {
676 case "fetch", "store", "search":
678 case "select", "examine":
679 // We don't send changes before having confirmed opening the mailbox, to prevent
680 // clients from trying to interpret changes when it considers there isn't a
681 // selected mailbox yet.
684 overflow, changes := c.comm.Get()
685 c.xapplyChanges(overflow, changes, true)
688 c.xbwritelinef(format, args...)
691func (c *conn) xwritelinef(format string, args ...any) {
692 c.xbwritelinef(format, args...)
696// Buffer line for write.
697func (c *conn) xbwritelinef(format string, args ...any) {
699 fmt.Fprintf(c.xbw, format, args...)
702func (c *conn) xflush() {
703 // If the connection is already broken, we're not going to write more.
709 xcheckf(err, "flush") // Should never happen, the Write caused by the Flush should panic on i/o error.
711 // If compression is enabled, we need to flush its stream.
713 // Note: Flush writes a sync message if there is nothing to flush. Ideally we
714 // wouldn't send that, but we would have to keep track of whether data needs to be
716 err := c.xflateWriter.Flush()
717 xcheckf(err, "flush deflate")
719 // The flate writer writes to a bufio.Writer, we must also flush that.
720 err = c.xflateBW.Flush()
721 xcheckf(err, "flush deflate writer")
725func (c *conn) parseCommand(tag *string, line string) (cmd string, p *parser) {
726 p = newParser(line, c)
732 return cmd, newParser(p.remainder(), c)
735func (c *conn) xreadliteral(size int64, sync bool) []byte {
739 buf := make([]byte, size)
741 if err := c.conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil {
742 c.log.Errorx("setting read deadline", err)
745 _, err := io.ReadFull(c.br, buf)
747 c.xbrokenf("reading literal: %s (%w)", err, errIO)
753var cleanClose struct{} // Sentinel value for panic/recover indicating clean close of connection.
755// serve handles a single IMAP connection on nc.
757// If xtls is set, immediate TLS should be enabled on the connection, unless
758// viaHTTP is set, which indicates TLS is already active with the connection coming
759// from the webserver with IMAP chosen through ALPN. activated. If viaHTTP is set,
760// the TLS config ddid not enable client certificate authentication. If xtls is
761// false and tlsConfig is set, STARTTLS may enable TLS later on.
763// If noRequireSTARTTLS is set, TLS is not required for authentication.
765// If accountAddress is not empty, it is the email address of the account to open
768// The connection is closed before returning.
769func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, xtls, noRequireSTARTTLS, viaHTTPS bool, preauthAddress string) {
771 if a, ok := nc.RemoteAddr().(*net.TCPAddr); ok {
774 // For tests and for imapserve.
775 remoteIP = net.ParseIP("127.0.0.10")
784 baseTLSConfig: tlsConfig,
786 noRequireSTARTTLS: noRequireSTARTTLS,
787 enabled: map[capability]bool{},
789 cmdStart: time.Now(),
791 var logmutex sync.Mutex
792 // Also see (and possibly update) c.logbg, for logging in a goroutine.
793 c.log = mlog.New("imapserver", nil).WithFunc(func() []slog.Attr {
795 defer logmutex.Unlock()
798 slog.Int64("cid", c.cid),
799 slog.Duration("delta", now.Sub(c.lastlog)),
802 if c.username != "" {
803 l = append(l, slog.String("username", c.username))
807 c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn)
808 // todo: tracing should be done on whatever comes out of c.br. the remote connection write a command plus data, and bufio can read it in one read, causing a command parser that sets the tracing level to data to have no effect. we are now typically logging sent messages, when mail clients append to the Sent mailbox.
809 c.br = bufio.NewReader(c.tr)
810 c.xtw = moxio.NewTraceWriter(c.log, "S: ", c)
811 c.xbw = bufio.NewWriter(c.xtw)
813 // Many IMAP connections use IDLE to wait for new incoming messages. We'll enable
814 // keepalive to get a higher chance of the connection staying alive, or otherwise
815 // detecting broken connections early.
818 tcpconn = nc.(*tls.Conn).NetConn()
820 if tc, ok := tcpconn.(*net.TCPConn); ok {
821 if err := tc.SetKeepAlivePeriod(5 * time.Minute); err != nil {
822 c.log.Errorx("setting keepalive period", err)
823 } else if err := tc.SetKeepAlive(true); err != nil {
824 c.log.Errorx("enabling keepalive", err)
828 c.log.Info("new connection",
829 slog.Any("remote", c.conn.RemoteAddr()),
830 slog.Any("local", c.conn.LocalAddr()),
831 slog.Bool("tls", xtls),
832 slog.Bool("viahttps", viaHTTPS),
833 slog.String("listener", listenerName))
836 err := c.conn.Close()
838 c.log.Debugx("closing connection", err)
841 // If changes for NOTIFY's SELECTED-DELAYED are still pending, we'll acknowledge
842 // their message removals so the files can be erased.
843 c.flushNotifyDelayed()
845 if c.account != nil {
847 err := c.account.Close()
848 c.xsanity(err, "close account")
854 if x == nil || x == cleanClose {
855 c.log.Info("connection closed")
856 } else if err, ok := x.(error); ok && isClosed(err) {
857 c.log.Infox("connection closed", err)
859 c.log.Error("unhandled panic", slog.Any("err", x))
861 metrics.PanicInc(metrics.Imapserver)
862 unhandledPanics.Add(1) // For tests.
866 if xtls && !viaHTTPS {
867 // Start TLS on connection. We perform the handshake explicitly, so we can set a
868 // timeout, do client certificate authentication, log TLS details afterwards.
869 c.xtlsHandshakeAndAuthenticate(c.conn)
873 case <-mox.Shutdown.Done():
875 c.xwritelinef("* BYE mox shutting down")
880 if !limiterConnectionrate.Add(c.remoteIP, time.Now(), 1) {
881 c.xwritelinef("* BYE connection rate from your ip or network too high, slow down please")
885 // If remote IP/network resulted in too many authentication failures, refuse to serve.
886 if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
887 metrics.AuthenticationRatelimitedInc("imap")
888 c.log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", c.remoteIP))
889 c.xwritelinef("* BYE too many auth failures")
893 if !limiterConnections.Add(c.remoteIP, time.Now(), 1) {
894 c.log.Debug("refusing connection due to many open connections", slog.Any("remoteip", c.remoteIP))
895 c.xwritelinef("* BYE too many open connections from your ip or network")
898 defer limiterConnections.Add(c.remoteIP, time.Now(), -1)
900 // We register and unregister the original connection, in case it c.conn is
901 // replaced with a TLS connection later on.
902 mox.Connections.Register(nc, "imap", listenerName)
903 defer mox.Connections.Unregister(nc)
905 if preauthAddress != "" {
906 acc, _, _, err := store.OpenEmail(c.log, preauthAddress, false)
908 c.log.Debugx("open account for preauth address", err, slog.String("address", preauthAddress))
909 c.xwritelinef("* BYE open account for address: %s", err)
912 c.username = preauthAddress
914 c.comm = store.RegisterComm(c.account)
917 if c.account != nil && !c.noPreauth {
918 c.state = stateAuthenticated
919 c.xwritelinef("* PREAUTH [CAPABILITY %s] mox imap welcomes %s", c.capabilities(), c.username)
921 c.xwritelinef("* OK [CAPABILITY %s] mox imap", c.capabilities())
924 // Ensure any pending loginAttempt is written before we stop.
926 if c.loginAttempt != nil {
927 store.LoginAttemptAdd(context.Background(), c.logbg(), *c.loginAttempt)
929 c.loginAttemptTime = time.Time{}
935 c.xflush() // For flushing errors, or commands that did not flush explicitly.
937 // Flush login attempt if it hasn't already been flushed by an ID command within 1s
938 // after authentication.
939 if c.loginAttempt != nil && (c.loginAttempt.UserAgent != "" || time.Since(c.loginAttemptTime) >= time.Second) {
940 store.LoginAttemptAdd(context.Background(), c.logbg(), *c.loginAttempt)
942 c.loginAttemptTime = time.Time{}
947// isClosed returns whether i/o failed, typically because the connection is closed.
948// For connection errors, we often want to generate fewer logs.
949func isClosed(err error) bool {
950 return errors.Is(err, errIO) || errors.Is(err, errProtocol) || mlog.IsClosed(err)
953// newLoginAttempt initializes a c.loginAttempt, for adding to the store after
954// filling in the results and other details.
955func (c *conn) newLoginAttempt(useTLS bool, authMech string) {
956 if c.loginAttempt != nil {
957 store.LoginAttemptAdd(context.Background(), c.logbg(), *c.loginAttempt)
960 c.loginAttemptTime = time.Now()
962 var state *tls.ConnectionState
963 if tc, ok := c.conn.(*tls.Conn); ok && useTLS {
964 v := tc.ConnectionState()
968 localAddr := c.conn.LocalAddr().String()
969 localIP, _, _ := net.SplitHostPort(localAddr)
974 c.loginAttempt = &store.LoginAttempt{
975 RemoteIP: c.remoteIP.String(),
977 TLS: store.LoginAttemptTLS(state),
979 UserAgent: c.userAgent, // May still be empty, to be filled in later.
981 Result: store.AuthError, // Replaced by caller.
985// makeTLSConfig makes a new tls config that is bound to the connection for
986// possible client certificate authentication.
987func (c *conn) makeTLSConfig() *tls.Config {
988 // We clone the config so we can set VerifyPeerCertificate below to a method bound
989 // to this connection. Earlier, we set session keys explicitly on the base TLS
990 // config, so they can be used for this connection too.
991 tlsConf := c.baseTLSConfig.Clone()
993 // Allow client certificate authentication, for use with the sasl "external"
994 // authentication mechanism.
995 tlsConf.ClientAuth = tls.RequestClientCert
997 // We verify the client certificate during the handshake. The TLS handshake is
998 // initiated explicitly for incoming connections and during starttls, so we can
999 // immediately extract the account name and address used for authentication.
1000 tlsConf.VerifyPeerCertificate = c.tlsClientAuthVerifyPeerCert
1005// tlsClientAuthVerifyPeerCert can be used as tls.Config.VerifyPeerCertificate, and
1006// sets authentication-related fields on conn. This is not called on resumed TLS
1008func (c *conn) tlsClientAuthVerifyPeerCert(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
1009 if len(rawCerts) == 0 {
1013 // If we had too many authentication failures from this IP, don't attempt
1014 // authentication. If this is a new incoming connetion, it is closed after the TLS
1016 if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
1020 cert, err := x509.ParseCertificate(rawCerts[0])
1022 c.log.Debugx("parsing tls client certificate", err)
1025 if err := c.tlsClientAuthVerifyPeerCertParsed(cert); err != nil {
1026 c.log.Debugx("verifying tls client certificate", err)
1027 return fmt.Errorf("verifying client certificate: %w", err)
1032// tlsClientAuthVerifyPeerCertParsed verifies a client certificate. Called both for
1033// fresh and resumed TLS connections.
1034func (c *conn) tlsClientAuthVerifyPeerCertParsed(cert *x509.Certificate) error {
1035 if c.account != nil {
1036 return fmt.Errorf("cannot authenticate with tls client certificate after previous authentication")
1039 // todo: it would be nice to postpone storing the loginattempt for tls pubkey auth until we have the ID command. but delaying is complicated because we can't get the tls information in this function. that's why we store the login attempt in a goroutine below, where it can can get a lock when accessing the tls connection only when this function has returned. we can't access c.loginAttempt (we would turn it into a slice) in a goroutine without adding more locking. for now we'll do without user-agent/id for tls pub key auth.
1040 c.newLoginAttempt(false, "tlsclientauth")
1042 // Get TLS connection state in goroutine because we are called while performing the
1043 // TLS handshake, which already has the tls connection locked.
1044 conn := c.conn.(*tls.Conn)
1045 la := *c.loginAttempt
1046 c.loginAttempt = nil
1047 logbg := c.logbg() // Evaluate attributes now, can't do it in goroutine.
1050 // In case of panic don't take the whole program down.
1053 c.log.Error("recover from panic", slog.Any("panic", x))
1055 metrics.PanicInc(metrics.Imapserver)
1059 state := conn.ConnectionState()
1060 la.TLS = store.LoginAttemptTLS(&state)
1061 store.LoginAttemptAdd(context.Background(), logbg, la)
1064 if la.Result == store.AuthSuccess {
1065 mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
1067 mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
1071 // For many failed auth attempts, slow down verification attempts.
1072 if c.authFailed > 3 && authFailDelay > 0 {
1073 mox.Sleep(mox.Context, time.Duration(c.authFailed-3)*authFailDelay)
1075 c.authFailed++ // Compensated on success.
1077 // On the 3rd failed authentication, start responding slowly. Successful auth will
1078 // cause fast responses again.
1079 if c.authFailed >= 3 {
1084 shabuf := sha256.Sum256(cert.RawSubjectPublicKeyInfo)
1085 fp := base64.RawURLEncoding.EncodeToString(shabuf[:])
1086 c.loginAttempt.TLSPubKeyFingerprint = fp
1087 pubKey, err := store.TLSPublicKeyGet(context.TODO(), fp)
1089 if err == bstore.ErrAbsent {
1090 c.loginAttempt.Result = store.AuthBadCredentials
1092 return fmt.Errorf("looking up tls public key with fingerprint %s: %v", fp, err)
1094 c.loginAttempt.LoginAddress = pubKey.LoginAddress
1096 // Verify account exists and still matches address. We don't check for account
1097 // login being disabled if preauth is disabled. In that case, sasl external auth
1098 // will be done before credentials can be used, and login disabled will be checked
1099 // then, where it will result in a more helpful error message.
1100 checkLoginDisabled := !pubKey.NoIMAPPreauth
1101 acc, accName, _, err := store.OpenEmail(c.log, pubKey.LoginAddress, checkLoginDisabled)
1102 c.loginAttempt.AccountName = accName
1104 if errors.Is(err, store.ErrLoginDisabled) {
1105 c.loginAttempt.Result = store.AuthLoginDisabled
1107 // note: we cannot send a more helpful error message to the client.
1108 return fmt.Errorf("opening account for address %s for public key %s: %w", pubKey.LoginAddress, fp, err)
1113 c.xsanity(err, "close account")
1116 c.loginAttempt.AccountName = acc.Name
1117 if acc.Name != pubKey.Account {
1118 return fmt.Errorf("tls client public key %s is for account %s, but email address %s is for account %s", fp, pubKey.Account, pubKey.LoginAddress, acc.Name)
1121 c.loginAttempt.Result = store.AuthSuccess
1124 c.noPreauth = pubKey.NoIMAPPreauth
1126 acc = nil // Prevent cleanup by defer.
1127 c.username = pubKey.LoginAddress
1128 c.comm = store.RegisterComm(c.account)
1129 c.log.Debug("tls client authenticated with client certificate",
1130 slog.String("fingerprint", fp),
1131 slog.String("username", c.username),
1132 slog.String("account", c.account.Name),
1133 slog.Any("remote", c.remoteIP))
1137// xtlsHandshakeAndAuthenticate performs the TLS handshake, and verifies a client
1138// certificate if present.
1139func (c *conn) xtlsHandshakeAndAuthenticate(conn net.Conn) {
1140 tlsConn := tls.Server(conn, c.makeTLSConfig())
1142 c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn)
1143 c.br = bufio.NewReader(c.tr)
1145 cidctx := context.WithValue(mox.Context, mlog.CidKey, c.cid)
1146 ctx, cancel := context.WithTimeout(cidctx, time.Minute)
1148 c.log.Debug("starting tls server handshake")
1149 if err := tlsConn.HandshakeContext(ctx); err != nil {
1150 c.xbrokenf("tls handshake: %s (%w)", err, errIO)
1154 cs := tlsConn.ConnectionState()
1155 if cs.DidResume && len(cs.PeerCertificates) > 0 {
1156 // Verify client after session resumption.
1157 err := c.tlsClientAuthVerifyPeerCertParsed(cs.PeerCertificates[0])
1159 c.xwritelinef("* BYE [ALERT] Error verifying client certificate after TLS session resumption: %s", err)
1160 c.xbrokenf("tls verify client certificate after resumption: %s (%w)", err, errIO)
1164 version, ciphersuite := moxio.TLSInfo(cs)
1165 attrs := []slog.Attr{
1166 slog.String("version", version),
1167 slog.String("ciphersuite", ciphersuite),
1168 slog.String("sni", cs.ServerName),
1169 slog.Bool("resumed", cs.DidResume),
1170 slog.Int("clientcerts", len(cs.PeerCertificates)),
1172 if c.account != nil {
1173 attrs = append(attrs,
1174 slog.String("account", c.account.Name),
1175 slog.String("username", c.username),
1178 c.log.Debug("tls handshake completed", attrs...)
1181func (c *conn) command() {
1182 var tag, cmd, cmdlow string
1188 metricIMAPCommands.WithLabelValues(c.cmdMetric, result).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second))
1191 logFields := []slog.Attr{
1192 slog.String("cmd", c.cmd),
1193 slog.Duration("duration", time.Since(c.cmdStart)),
1198 if x == nil || x == cleanClose {
1199 c.log.Debug("imap command done", logFields...)
1201 if x == cleanClose {
1202 // If compression was enabled, we flush & close the deflate stream.
1204 // Note: Close and flush can Write and may panic with an i/o error.
1205 if err := c.xflateWriter.Close(); err != nil {
1206 c.log.Debugx("close deflate writer", err)
1207 } else if err := c.xflateBW.Flush(); err != nil {
1208 c.log.Debugx("flush deflate buffer", err)
1216 err, ok := x.(error)
1218 c.log.Error("imap command panic", append([]slog.Attr{slog.Any("panic", x)}, logFields...)...)
1223 var sxerr syntaxError
1225 var serr serverError
1227 c.log.Infox("imap command ioerror", err, logFields...)
1229 if errors.Is(err, errProtocol) {
1233 } else if errors.As(err, &sxerr) {
1234 result = "badsyntax"
1236 // Other side is likely speaking something else than IMAP, send error message and
1237 // stop processing because there is a good chance whatever they sent has multiple
1239 c.xwritelinef("* BYE please try again speaking imap")
1240 c.xbrokenf("not speaking imap (%w)", errIO)
1242 c.log.Debugx("imap command syntax error", sxerr.err, logFields...)
1243 c.log.Info("imap syntax error", slog.String("lastline", c.lastLine))
1244 fatal := strings.HasSuffix(c.lastLine, "+}")
1246 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
1247 c.log.Check(err, "setting write deadline")
1249 if sxerr.line != "" {
1250 c.xbwritelinef("%s", sxerr.line)
1253 if sxerr.code != "" {
1254 code = "[" + sxerr.code + "] "
1256 c.xbwriteresultf("%s BAD %s%s unrecognized syntax/command: %v", tag, code, cmd, sxerr.errmsg)
1259 panic(fmt.Errorf("aborting connection after syntax error for command with non-sync literal: %w", errProtocol))
1261 } else if errors.As(err, &serr) {
1262 result = "servererror"
1263 c.log.Errorx("imap command server error", err, logFields...)
1265 c.xbwriteresultf("%s NO %s %v", tag, cmd, err)
1266 } else if errors.As(err, &uerr) {
1267 result = "usererror"
1268 c.log.Debugx("imap command user error", err, logFields...)
1269 if uerr.code != "" {
1270 c.xbwriteresultf("%s NO [%s] %s %v", tag, uerr.code, cmd, err)
1272 c.xbwriteresultf("%s NO %s %v", tag, cmd, err)
1275 // Other type of panic, we pass it on, aborting the connection.
1277 c.log.Errorx("imap command panic", err, logFields...)
1284 // If NOTIFY is enabled, we wait for either a line (with a command) from the
1285 // client, or a change event. If we see a line, we continue below as for the
1286 // non-NOTIFY case, parsing the command.
1288 if c.notify != nil {
1292 case le := <-c.lineChan():
1294 if err := le.err; err != nil {
1295 if errors.Is(err, os.ErrDeadlineExceeded) {
1296 err := c.conn.SetDeadline(time.Now().Add(10 * time.Second))
1297 c.log.Check(err, "setting write deadline")
1298 c.xwritelinef("* BYE inactive")
1301 if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) {
1302 c.xbrokenf("%s (%w)", err, errIO)
1309 case <-c.comm.Pending:
1310 overflow, changes := c.comm.Get()
1311 c.xapplyChanges(overflow, changes, false)
1314 case <-mox.Shutdown.Done():
1316 c.xwritelinef("* BYE shutting down")
1317 c.xbrokenf("shutting down (%w)", errIO)
1321 // Reset the write deadline. In case of little activity, with a command timeout of
1322 // 30 minutes, we have likely passed it.
1323 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
1324 c.log.Check(err, "setting write deadline")
1326 // Without NOTIFY, we just read a line.
1327 line = c.xreadline(true)
1329 cmd, p = c.parseCommand(&tag, line)
1330 cmdlow = strings.ToLower(cmd)
1332 c.cmdStart = time.Now()
1333 c.cmdMetric = "(unrecognized)"
1336 case <-mox.Shutdown.Done():
1338 c.xwritelinef("* BYE shutting down")
1339 c.xbrokenf("shutting down (%w)", errIO)
1343 fn := commands[cmdlow]
1345 xsyntaxErrorf("unknown command %q", cmd)
1350 // Check if command is allowed in this state.
1351 if _, ok1 := commandsStateAny[cmdlow]; ok1 {
1352 } else if _, ok2 := commandsStateNotAuthenticated[cmdlow]; ok2 && c.state == stateNotAuthenticated {
1353 } else if _, ok3 := commandsStateAuthenticated[cmdlow]; ok3 && c.state == stateAuthenticated || c.state == stateSelected {
1354 } else if _, ok4 := commandsStateSelected[cmdlow]; ok4 && c.state == stateSelected {
1355 } else if ok1 || ok2 || ok3 || ok4 {
1356 xuserErrorf("not allowed in this connection state")
1358 xserverErrorf("unrecognized command")
1362 if _, ok := commandsSequence[cmdlow]; ok && c.uidonly {
1363 xsyntaxCodeErrorf("UIDREQUIRED", "cannot use message sequence numbers with uidonly")
1369func (c *conn) broadcast(changes []store.Change) {
1370 if len(changes) == 0 {
1373 c.log.Debug("broadcast changes", slog.Any("changes", changes))
1374 c.comm.Broadcast(changes)
1377// matchStringer matches a string against reference + mailbox patterns.
1378type matchStringer interface {
1379 MatchString(s string) bool
1382type noMatch struct{}
1384// MatchString for noMatch always returns false.
1385func (noMatch) MatchString(s string) bool {
1389// xmailboxPatternMatcher returns a matcher for mailbox names given the reference and patterns.
1390// Patterns can include "%" and "*", matching any character excluding and including a slash respectively.
1391func xmailboxPatternMatcher(ref string, patterns []string) matchStringer {
1392 if strings.HasPrefix(ref, "/") {
1397 for _, pat := range patterns {
1398 if strings.HasPrefix(pat, "/") {
1404 s = path.Join(ref, pat)
1407 // Fix casing for all Inbox paths.
1408 first := strings.SplitN(s, "/", 2)[0]
1409 if strings.EqualFold(first, "Inbox") {
1410 s = "Inbox" + s[len("Inbox"):]
1415 for _, c := range s {
1418 } else if c == '*' {
1421 rs += regexp.QuoteMeta(string(c))
1424 subs = append(subs, rs)
1430 rs := "^(" + strings.Join(subs, "|") + ")$"
1431 re, err := regexp.Compile(rs)
1432 xcheckf(err, "compiling regexp for mailbox patterns")
1436func (c *conn) sequence(uid store.UID) msgseq {
1438 panic("sequence with uidonly")
1440 return uidSearch(c.uids, uid)
1443func uidSearch(uids []store.UID, uid store.UID) msgseq {
1450 return msgseq(i + 1)
1460func (c *conn) xsequence(uid store.UID) msgseq {
1462 panic("xsequence with uidonly")
1464 seq := c.sequence(uid)
1466 xserverErrorf("unknown uid %d (%w)", uid, errProtocol)
1471func (c *conn) sequenceRemove(seq msgseq, uid store.UID) {
1473 panic("sequenceRemove with uidonly")
1476 if c.uids[i] != uid {
1477 xserverErrorf("got uid %d at msgseq %d, expected uid %d", uid, seq, c.uids[i])
1479 copy(c.uids[i:], c.uids[i+1:])
1480 c.uids = c.uids[:c.exists-1]
1482 c.checkUIDs(c.uids, true)
1485// add uid to session, through c.uidnext, and if uidonly isn't enabled to c.uids.
1486// care must be taken that pending changes are fetched while holding the account
1487// wlock, and applied before adding this uid, because those pending changes may
1488// contain another new uid that has to be added first.
1489func (c *conn) uidAppend(uid store.UID) {
1491 if uid < c.uidnext {
1492 panic(fmt.Sprintf("new uid %d < uidnext %d", uid, c.uidnext))
1499 if uidSearch(c.uids, uid) > 0 {
1500 xserverErrorf("uid already present (%w)", errProtocol)
1502 if c.exists > 0 && uid < c.uids[c.exists-1] {
1503 xserverErrorf("new uid %d is smaller than last uid %d (%w)", uid, c.uids[c.exists-1], errProtocol)
1507 c.uids = append(c.uids, uid)
1508 c.checkUIDs(c.uids, true)
1511// sanity check that uids are in ascending order.
1512func (c *conn) checkUIDs(uids []store.UID, checkExists bool) {
1517 if checkExists && uint32(len(uids)) != c.exists {
1518 panic(fmt.Sprintf("exists %d does not match len(uids) %d", c.exists, len(c.uids)))
1521 for i, uid := range uids {
1522 if uid == 0 || i > 0 && uid <= uids[i-1] {
1523 xserverErrorf("bad uids %v", uids)
1528func slicesAny[T any](l []T) []any {
1529 r := make([]any, len(l))
1530 for i, v := range l {
1536// newCachedLastUID returns a method that returns the highest uid for a mailbox,
1537// for interpretation of "*". If mailboxID is for the selected mailbox, the UIDs
1538// visible in the session are taken into account. If there is no UID, 0 is
1539// returned. If an error occurs, xerrfn is called, which should not return.
1540func (c *conn) newCachedLastUID(tx *bstore.Tx, mailboxID int64, xerrfn func(err error)) func() store.UID {
1543 return func() store.UID {
1547 if c.mailboxID == mailboxID {
1552 return c.uids[c.exists-1]
1555 q := bstore.QueryTx[store.Message](tx)
1556 q.FilterNonzero(store.Message{MailboxID: mailboxID})
1557 q.FilterEqual("Expunged", false)
1558 if c.mailboxID == mailboxID {
1559 q.FilterLess("UID", c.uidnext)
1564 if err == bstore.ErrAbsent {
1570 panic(err) // xerrfn should have called panic.
1578// xnumSetEval evaluates nums to uids given the current session state and messages
1579// in the selected mailbox. The returned UIDs are sorted, without duplicates.
1580func (c *conn) xnumSetEval(tx *bstore.Tx, isUID bool, nums numSet) []store.UID {
1581 if nums.searchResult {
1582 // UIDs that do not exist can be ignored.
1587 // Update previously stored UIDs. Some may have been deleted.
1588 // Once deleted a UID will never come back, so we'll just remove those uids.
1590 var uids []store.UID
1591 if len(c.searchResult) > 0 {
1592 q := bstore.QueryTx[store.Message](tx)
1593 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
1594 q.FilterEqual("Expunged", false)
1595 q.FilterEqual("UID", slicesAny(c.searchResult)...)
1597 for m, err := range q.All() {
1598 xcheckf(err, "looking up messages from search result")
1599 uids = append(uids, m.UID)
1602 c.searchResult = uids
1605 for _, uid := range c.searchResult {
1606 if uidSearch(c.uids, uid) > 0 {
1607 c.searchResult[o] = uid
1611 c.searchResult = c.searchResult[:o]
1613 return c.searchResult
1617 uids := map[store.UID]struct{}{}
1619 // Sequence numbers that don't exist, or * on an empty mailbox, should result in a BAD response.
../rfc/9051:7018
1620 for _, r := range nums.ranges {
1624 xsyntaxErrorf("invalid seqset * on empty mailbox")
1626 ia = int(c.exists) - 1
1628 ia = int(r.first.number - 1)
1629 if ia >= int(c.exists) {
1630 xsyntaxErrorf("msgseq %d not in mailbox", r.first.number)
1634 uids[c.uids[ia]] = struct{}{}
1639 ib = int(c.exists) - 1
1641 ib = int(r.last.number - 1)
1642 if ib >= int(c.exists) {
1643 xsyntaxErrorf("msgseq %d not in mailbox", r.last.number)
1649 for _, uid := range c.uids[ia : ib+1] {
1650 uids[uid] = struct{}{}
1653 return slices.Sorted(maps.Keys(uids))
1656 // UIDs that do not exist can be ignored.
1661 uids := map[store.UID]struct{}{}
1664 xlastUID := c.newCachedLastUID(tx, c.mailboxID, func(xerr error) { xuserErrorf("%s", xerr) })
1665 for _, r := range nums.xinterpretStar(xlastUID).ranges {
1666 q := bstore.QueryTx[store.Message](tx)
1667 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
1668 q.FilterEqual("Expunged", false)
1670 q.FilterEqual("UID", r.first.number)
1672 q.FilterGreaterEqual("UID", r.first.number)
1673 q.FilterLessEqual("UID", r.last.number)
1675 q.FilterLess("UID", c.uidnext)
1677 for m, err := range q.All() {
1678 xcheckf(err, "enumerating uids")
1679 uids[m.UID] = struct{}{}
1682 return slices.Sorted(maps.Keys(uids))
1685 for _, r := range nums.ranges {
1691 uida := store.UID(r.first.number)
1693 uida = c.uids[c.exists-1]
1696 uidb := store.UID(last.number)
1698 uidb = c.uids[c.exists-1]
1702 uida, uidb = uidb, uida
1705 // Binary search for uida.
1710 if uida < c.uids[m] {
1712 } else if uida > c.uids[m] {
1719 for _, uid := range c.uids[s:] {
1720 if uid >= uida && uid <= uidb {
1721 uids[uid] = struct{}{}
1722 } else if uid > uidb {
1727 return slices.Sorted(maps.Keys(uids))
1730func (c *conn) ok(tag, cmd string) {
1731 c.xbwriteresultf("%s OK %s done", tag, cmd)
1735// xcheckmailboxname checks if name is valid, returning an INBOX-normalized name.
1736// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.
1737// Name is invalid if it contains leading/trailing/double slashes, or when it isn't
1738// unicode-normalized, or when empty or has special characters.
1739func xcheckmailboxname(name string, allowInbox bool) string {
1740 name, isinbox, err := store.CheckMailboxName(name, allowInbox)
1742 xuserErrorf("special mailboxname Inbox not allowed")
1743 } else if err != nil {
1744 xusercodeErrorf("CANNOT", "%s", err)
1749// Lookup mailbox by name.
1750// If the mailbox does not exist, panic is called with a user error.
1751// Must be called with account rlock held.
1752func (c *conn) xmailbox(tx *bstore.Tx, name string, missingErrCode string) store.Mailbox {
1753 mb, err := c.account.MailboxFind(tx, name)
1754 xcheckf(err, "finding mailbox")
1756 // missingErrCode can be empty, or e.g. TRYCREATE or ALREADYEXISTS.
1757 xusercodeErrorf(missingErrCode, "%w", store.ErrUnknownMailbox)
1762// Lookup mailbox by ID.
1763// If the mailbox does not exist, panic is called with a user error.
1764// Must be called with account rlock held.
1765func (c *conn) xmailboxID(tx *bstore.Tx, id int64) store.Mailbox {
1766 mb, err := store.MailboxID(tx, id)
1767 if err == bstore.ErrAbsent {
1768 xuserErrorf("%w", store.ErrUnknownMailbox)
1769 } else if err == store.ErrMailboxExpunged {
1771 xusercodeErrorf("NONEXISTENT", "mailbox has been deleted")
1776// Apply changes to our session state.
1777// Should not be called while holding locks, as changes are written to client connections, which can block.
1778// Does not flush output.
1779func (c *conn) xapplyChanges(overflow bool, changes []store.Change, sendDelayed bool) {
1780 // If more changes were generated than we can process, we send a
1783 if c.notify != nil && len(c.notify.Delayed) > 0 {
1784 changes = append(c.notify.Delayed, changes...)
1786 c.flushChanges(changes)
1787 // We must not send any more unsolicited untagged responses to the client for
1789 c.notify = ¬ify{}
1790 c.xbwritelinef("* OK [NOTIFICATIONOVERFLOW] out of sync after too many pending changes")
1794 // applyChanges for IDLE and NOTIFY. When explicitly in IDLE while NOTIFY is
1796 if c.notify != nil {
1797 c.xapplyChangesNotify(changes, sendDelayed)
1800 if len(changes) == 0 {
1804 // Even in the case of a panic (e.g. i/o errors), we must mark removals as seen.
1805 origChanges := changes
1807 for _, change := range origChanges {
1808 if ch, ok := change.(store.ChangeRemoveUIDs); ok {
1809 c.comm.RemovalSeen(ch)
1814 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
1815 c.log.Check(err, "setting write deadline")
1817 c.log.Debug("applying changes", slog.Any("changes", changes))
1819 // Only keep changes for the selected mailbox, and changes that are always relevant.
1820 var n []store.Change
1821 for _, change := range changes {
1823 switch ch := change.(type) {
1824 case store.ChangeAddUID:
1826 case store.ChangeRemoveUIDs:
1828 case store.ChangeFlags:
1830 case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription, store.ChangeRemoveSubscription:
1831 n = append(n, change)
1833 case store.ChangeAnnotation:
1834 // note: annotations may have a mailbox associated with them, but we pass all
1837 if c.enabled[capMetadata] {
1838 n = append(n, change)
1841 case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords, store.ChangeThread:
1843 panic(fmt.Errorf("missing case for %#v", change))
1845 if c.state == stateSelected && mbID == c.mailboxID {
1846 n = append(n, change)
1851 qresync := c.enabled[capQresync]
1852 condstore := c.enabled[capCondstore]
1855 for i < len(changes) {
1856 // First process all new uids. So we only send a single EXISTS.
1857 var adds []store.ChangeAddUID
1858 for ; i < len(changes); i++ {
1859 ch, ok := changes[i].(store.ChangeAddUID)
1864 adds = append(adds, ch)
1867 // Write the exists, and the UID and flags as well. Hopefully the client waits for
1868 // long enough after the EXISTS to see these messages, and doesn't request them
1869 // again with a FETCH.
1870 c.xbwritelinef("* %d EXISTS", c.exists)
1871 for _, add := range adds {
1872 var modseqStr string
1874 modseqStr = fmt.Sprintf(" MODSEQ (%d)", add.ModSeq.Client())
1878 c.xbwritelinef("* %d UIDFETCH (FLAGS %s%s)", add.UID, flaglist(add.Flags, add.Keywords).pack(c), modseqStr)
1880 seq := c.xsequence(add.UID)
1881 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, add.UID, flaglist(add.Flags, add.Keywords).pack(c), modseqStr)
1887 change := changes[i]
1890 switch ch := change.(type) {
1891 case store.ChangeRemoveUIDs:
1892 var vanishedUIDs numSet
1893 for _, uid := range ch.UIDs {
1897 vanishedUIDs.append(uint32(uid))
1901 seq := c.xsequence(uid)
1902 c.sequenceRemove(seq, uid)
1904 vanishedUIDs.append(uint32(uid))
1906 c.xbwritelinef("* %d EXPUNGE", seq)
1909 if !vanishedUIDs.empty() {
1911 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
1912 c.xbwritelinef("* VANISHED %s", s)
1916 case store.ChangeFlags:
1917 var modseqStr string
1919 modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client())
1923 c.xbwritelinef("* %d UIDFETCH (FLAGS %s%s)", ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
1925 // The uid can be unknown if we just expunged it while another session marked it as deleted just before.
1926 seq := c.sequence(ch.UID)
1930 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
1933 case store.ChangeRemoveMailbox:
1934 // Only announce \NonExistent to modern clients, otherwise they may ignore the
1935 // unrecognized \NonExistent and interpret this as a newly created mailbox, while
1936 // the goal was to remove it...
1937 if c.enabled[capIMAP4rev2] {
1938 c.xbwritelinef(`* LIST (\NonExistent) "/" %s`, mailboxt(ch.Name).pack(c))
1941 case store.ChangeAddMailbox:
1942 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), mailboxt(ch.Mailbox.Name).pack(c))
1944 case store.ChangeRenameMailbox:
1947 if c.enabled[capIMAP4rev2] {
1948 oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(ch.OldName).pack(c))
1950 c.xbwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), mailboxt(ch.NewName).pack(c), oldname)
1952 case store.ChangeAddSubscription:
1953 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.ListFlags...), " "), mailboxt(ch.MailboxName).pack(c))
1955 case store.ChangeRemoveSubscription:
1956 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.ListFlags, " "), mailboxt(ch.MailboxName).pack(c))
1958 case store.ChangeAnnotation:
1960 c.xbwritelinef(`* METADATA %s %s`, mailboxt(ch.MailboxName).pack(c), astring(ch.Key).pack(c))
1963 panic(fmt.Sprintf("internal error, missing case for %#v", change))
1968// xapplyChangesNotify is like xapplyChanges, but for NOTIFY, with configurable
1969// mailboxes to notify about, and configurable events to send, including which
1970// fetch attributes to return. All calls must go through xapplyChanges, for overflow
1972func (c *conn) xapplyChangesNotify(changes []store.Change, sendDelayed bool) {
1973 if sendDelayed && len(c.notify.Delayed) > 0 {
1974 changes = append(c.notify.Delayed, changes...)
1975 c.notify.Delayed = nil
1978 if len(changes) == 0 {
1982 // Even in the case of a panic (e.g. i/o errors), we must mark removals as seen.
1983 // For selected-delayed, we may have postponed handling the message, so we call
1984 // RemovalSeen when handling a change, and mark how far we got, so we only process
1985 // changes that we haven't processed yet.
1986 unhandled := changes
1988 for _, change := range unhandled {
1989 if ch, ok := change.(store.ChangeRemoveUIDs); ok {
1990 c.comm.RemovalSeen(ch)
1995 c.log.Debug("applying notify changes", slog.Any("changes", changes))
1997 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
1998 c.log.Check(err, "setting write deadline")
2000 qresync := c.enabled[capQresync]
2001 condstore := c.enabled[capCondstore]
2003 // Prepare for providing a read-only transaction on first-use, for MessageNew fetch
2008 err := tx.Rollback()
2009 c.log.Check(err, "rolling back tx")
2012 xtx := func() *bstore.Tx {
2018 tx, err = c.account.DB.Begin(context.TODO(), false)
2023 // On-demand mailbox lookups, with cache.
2024 mailboxes := map[int64]store.Mailbox{}
2025 xmailbox := func(id int64) store.Mailbox {
2026 if mb, ok := mailboxes[id]; ok {
2029 mb := store.Mailbox{ID: id}
2030 err := xtx().Get(&mb)
2031 xcheckf(err, "get mailbox")
2036 // Keep track of last command, to close any open message file (for fetching
2037 // attributes) in case of a panic.
2046 for index, change := range changes {
2047 switch ch := change.(type) {
2048 case store.ChangeAddUID:
2050 // todo:
../rfc/5465:525 group ChangeAddUID for the same mailbox, so we can send a single EXISTS. useful for imports.
2052 mb := xmailbox(ch.MailboxID)
2053 ms, ev, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMessageNew)
2058 // For non-selected mailbox, send STATUS with UIDNEXT, MESSAGES. And HIGESTMODSEQ
2060 // There is no mention of UNSEEN for MessageNew, but clients will want to show a
2061 // new "unread messages" count, and they will have to understand it since
2062 // FlagChange is specified as sending UNSEEN.
2063 if mb.ID != c.mailboxID {
2064 if condstore || qresync {
2065 c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d HIGHESTMODSEQ %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UID+1, ch.MessageCountIMAP, ch.ModSeq, ch.Unseen)
2067 c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UID+1, ch.MessageCountIMAP, ch.Unseen)
2072 // Delay sending all message events, we want to prevent synchronization issues
2074 if ms.Kind == mbspecSelectedDelayed && !sendDelayed {
2075 c.notify.Delayed = append(c.notify.Delayed, change)
2082 c.xbwritelinef("* %d EXISTS", c.exists)
2084 // If client did not specify attributes, we'll send the defaults.
2085 if len(ev.FetchAtt) == 0 {
2086 var modseqStr string
2088 modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client())
2090 // NOTIFY does not specify the default fetch attributes to return, we send UID and
2094 c.xbwritelinef("* %d UIDFETCH (FLAGS %s%s)", ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
2096 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", c.xsequence(ch.UID), ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
2101 // todo:
../rfc/5465:543 mark messages as \seen after processing if client didn't use the .PEEK-variants.
2102 cmd = &fetchCmd{conn: c, isUID: true, rtx: xtx(), mailboxID: ch.MailboxID, uid: ch.UID}
2103 data, err := cmd.process(ev.FetchAtt)
2105 // There is no good way to notify the client about errors. We continue below to
2106 // send a FETCH with just the UID. And we send an untagged NO in the hope a client
2107 // developer sees the message.
2108 c.log.Errorx("generating notify fetch response", err, slog.Int64("mailboxid", ch.MailboxID), slog.Any("uid", ch.UID))
2109 c.xbwritelinef("* NO generating notify fetch response: %s", err.Error())
2110 // Always add UID, also for uidonly, to ensure a non-empty list.
2111 data = listspace{bare("UID"), number(ch.UID)}
2115 fmt.Fprintf(cmd.conn.xbw, "* %d UIDFETCH ", ch.UID)
2117 fmt.Fprintf(cmd.conn.xbw, "* %d FETCH ", c.xsequence(ch.UID))
2120 defer c.xtracewrite(mlog.LevelTracedata)()
2121 data.xwriteTo(cmd.conn, cmd.conn.xbw)
2122 c.xtracewrite(mlog.LevelTrace) // Restore.
2123 cmd.conn.xbw.Write([]byte("\r\n"))
2129 case store.ChangeRemoveUIDs:
2131 mb := xmailbox(ch.MailboxID)
2132 ms, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMessageExpunge)
2134 unhandled = changes[index+1:]
2135 c.comm.RemovalSeen(ch)
2139 // For non-selected mailboxes, we send STATUS with at least UIDNEXT and MESSAGES.
2141 // In case of QRESYNC, we send HIGHESTMODSEQ. Also for CONDSTORE, which isn't
2144 // There is no mention of UNSEEN, but clients will want to show a new "unread
2145 // messages" count, and they can parse it since FlagChange is specified as sending
2147 if mb.ID != c.mailboxID {
2148 unhandled = changes[index+1:]
2149 c.comm.RemovalSeen(ch)
2150 if condstore || qresync {
2151 c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d HIGHESTMODSEQ %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UIDNext, ch.MessageCountIMAP, ch.ModSeq, ch.Unseen)
2153 c.xbwritelinef("* STATUS %s (UIDNEXT %d MESSAGES %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UIDNext, ch.MessageCountIMAP, ch.Unseen)
2158 // Delay sending all message events, we want to prevent synchronization issues
2160 if ms.Kind == mbspecSelectedDelayed && !sendDelayed {
2161 unhandled = changes[index+1:] // We'll call RemovalSeen in the future.
2162 c.notify.Delayed = append(c.notify.Delayed, change)
2166 unhandled = changes[index+1:]
2167 c.comm.RemovalSeen(ch)
2169 var vanishedUIDs numSet
2170 for _, uid := range ch.UIDs {
2174 vanishedUIDs.append(uint32(uid))
2178 seq := c.xsequence(uid)
2179 c.sequenceRemove(seq, uid)
2181 vanishedUIDs.append(uint32(uid))
2183 c.xbwritelinef("* %d EXPUNGE", seq)
2186 if !vanishedUIDs.empty() {
2188 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
2189 c.xbwritelinef("* VANISHED %s", s)
2193 case store.ChangeFlags:
2195 mb := xmailbox(ch.MailboxID)
2196 ms, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventFlagChange)
2199 } else if mb.ID != c.mailboxID {
2202 // We include UNSEEN, so clients can update the number of unread messages.
../rfc/5465:479
2203 if condstore || qresync {
2204 c.xbwritelinef("* STATUS %s (HIGHESTMODSEQ %d UIDVALIDITY %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.ModSeq, ch.UIDValidity, ch.Unseen)
2206 c.xbwritelinef("* STATUS %s (UIDVALIDITY %d UNSEEN %d)", mailboxt(mb.Name).pack(c), ch.UIDValidity, ch.Unseen)
2211 // Delay sending all message events, we want to prevent synchronization issues
2213 if ms.Kind == mbspecSelectedDelayed && !sendDelayed {
2214 c.notify.Delayed = append(c.notify.Delayed, change)
2218 // The uid can be unknown if we just expunged it while another session marked it as deleted just before.
2221 seq = c.sequence(ch.UID)
2227 var modseqStr string
2229 modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client())
2234 c.xbwritelinef("* %d UIDFETCH (FLAGS %s%s)", ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
2236 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
2239 case store.ChangeThread:
2243 case store.ChangeRemoveMailbox:
2244 mb := xmailbox(ch.MailboxID)
2245 _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxName)
2251 c.xbwritelinef(`* LIST (\NonExistent) "/" %s`, mailboxt(ch.Name).pack(c))
2253 case store.ChangeAddMailbox:
2254 mb := xmailbox(ch.Mailbox.ID)
2255 _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxName)
2259 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), mailboxt(ch.Mailbox.Name).pack(c))
2261 case store.ChangeRenameMailbox:
2262 mb := xmailbox(ch.MailboxID)
2263 _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxName)
2268 oldname := fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(ch.OldName).pack(c))
2269 c.xbwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), mailboxt(ch.NewName).pack(c), oldname)
2272 case store.ChangeAddSubscription:
2273 _, _, ok := c.notify.match(c, xtx, 0, ch.MailboxName, eventSubscriptionChange)
2277 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.ListFlags...), " "), mailboxt(ch.MailboxName).pack(c))
2279 case store.ChangeRemoveSubscription:
2280 _, _, ok := c.notify.match(c, xtx, 0, ch.MailboxName, eventSubscriptionChange)
2285 c.xbwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.ListFlags, " "), mailboxt(ch.MailboxName).pack(c))
2287 case store.ChangeMailboxCounts:
2290 case store.ChangeMailboxSpecialUse:
2291 // todo: can we send special-use flags as part of an untagged LIST response?
2294 case store.ChangeMailboxKeywords:
2296 mb := xmailbox(ch.MailboxID)
2297 ms, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventFlagChange)
2300 } else if mb.ID != c.mailboxID {
2304 // Delay sending all message events, we want to prevent synchronization issues
2306 // This change is about mailbox keywords, but it's specified under the FlagChange
2309 if ms.Kind == mbspecSelectedDelayed && !sendDelayed {
2310 c.notify.Delayed = append(c.notify.Delayed, change)
2315 if len(ch.Keywords) > 0 {
2316 keywords = " " + strings.Join(ch.Keywords, " ")
2318 c.xbwritelinef(`* FLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent%s)`, keywords)
2320 case store.ChangeAnnotation:
2321 // Client does not have to enable METADATA/METADATA-SERVER. Just asking for these
2322 // events is enough.
2325 if ch.MailboxID == 0 {
2327 _, _, ok := c.notify.match(c, xtx, 0, "", eventServerMetadataChange)
2333 mb := xmailbox(ch.MailboxID)
2334 _, _, ok := c.notify.match(c, xtx, mb.ID, mb.Name, eventMailboxMetadataChange)
2343 c.xbwritelinef(`* METADATA %s %s`, mailboxt(ch.MailboxName).pack(c), astring(ch.Key).pack(c))
2346 panic(fmt.Sprintf("internal error, missing case for %#v", change))
2350 // If we have too many delayed changes, we will warn about notification overflow,
2352 if len(c.notify.Delayed) > selectedDelayedChangesMax {
2353 l := c.notify.Delayed
2354 c.notify.Delayed = nil
2357 c.notify = ¬ify{}
2358 c.xbwritelinef("* OK [NOTIFICATIONOVERFLOW] out of sync after too many pending changes for selected mailbox")
2362// Capability returns the capabilities this server implements and currently has
2363// available given the connection state.
2366func (c *conn) cmdCapability(tag, cmd string, p *parser) {
2372 caps := c.capabilities()
2375 c.xbwritelinef("* CAPABILITY %s", caps)
2379// capabilities returns non-empty string with available capabilities based on connection state.
2380// For use in cmdCapability and untagged OK responses on connection start, login and authenticate.
2381func (c *conn) capabilities() string {
2382 caps := serverCapabilities
2384 // We only allow starting without TLS when explicitly configured, in violation of RFC.
2385 if !c.tls && c.baseTLSConfig != nil {
2388 if c.tls || c.noRequireSTARTTLS {
2389 caps += " AUTH=PLAIN"
2391 caps += " LOGINDISABLED"
2393 if c.tls && len(c.conn.(*tls.Conn).ConnectionState().PeerCertificates) > 0 && !c.viaHTTPS {
2394 caps += " AUTH=EXTERNAL"
2399// No op, but useful for retrieving pending changes as untagged responses, e.g. of
2403func (c *conn) cmdNoop(tag, cmd string, p *parser) {
2411// Logout, after which server closes the connection.
2414func (c *conn) cmdLogout(tag, cmd string, p *parser) {
2421 c.state = stateNotAuthenticated
2423 c.xbwritelinef("* BYE thanks")
2428// Clients can use ID to tell the server which software they are using. Servers can
2429// respond with their version. For statistics/logging/debugging purposes.
2432func (c *conn) cmdID(tag, cmd string, p *parser) {
2437 var params map[string]string
2440 params = map[string]string{}
2442 if len(params) > 0 {
2448 if _, ok := params[k]; ok {
2449 xsyntaxErrorf("duplicate key %q", k)
2452 values = append(values, fmt.Sprintf("%s=%q", k, v))
2459 c.userAgent = strings.Join(values, " ")
2461 // The ID command is typically sent soon after authentication. So we've prepared
2462 // the LoginAttempt and write it now.
2463 if c.loginAttempt != nil {
2464 c.loginAttempt.UserAgent = c.userAgent
2465 store.LoginAttemptAdd(context.Background(), c.logbg(), *c.loginAttempt)
2466 c.loginAttempt = nil
2467 c.loginAttemptTime = time.Time{}
2470 // We just log the client id.
2471 c.log.Info("client id", slog.Any("params", params))
2475 if c.state == stateAuthenticated || c.state == stateSelected {
2476 c.xbwritelinef(`* ID ("name" "mox" "version" %s)`, string0(moxvar.Version).pack(c))
2478 c.xbwritelinef(`* ID ("name" "mox")`)
2483// Compress enables compression on the connection. Deflate is the only algorithm
2484// specified. TLS doesn't do compression nowadays, so we don't have to check for that.
2486// Status: Authenticated. The RFC doesn't mention this in prose, but the command is
2487// added to ABNF production rule "command-auth".
2488func (c *conn) cmdCompress(tag, cmd string, p *parser) {
2496 // Will do compression only once.
2499 xusercodeErrorf("COMPRESSIONACTIVE", "compression already active with previous compress command")
2502 if !strings.EqualFold(alg, "deflate") {
2503 xuserErrorf("compression algorithm not supported")
2506 // We must flush now, before we initialize flate.
2507 c.log.Debug("compression enabled")
2510 c.xflateBW = bufio.NewWriter(c)
2511 fw0, err := flate.NewWriter(c.xflateBW, flate.DefaultCompression)
2512 xcheckf(err, "deflate") // Cannot happen.
2513 xfw := moxio.NewFlateWriter(fw0)
2516 c.xflateWriter = xfw
2517 c.xtw = moxio.NewTraceWriter(c.log, "S: ", c.xflateWriter)
2518 c.xbw = bufio.NewWriter(c.xtw) // The previous c.xbw will not have buffered data.
2520 rc := xprefixConn(c.conn, c.br) // c.br may contain buffered data.
2521 // We use the special partial reader. Some clients write commands and flush the
2522 // buffer in "partial flush" mode instead of "sync flush" mode. The "sync flush"
2523 // mode emits an explicit zero-length data block that triggers the Go stdlib flate
2524 // reader to return data to us. It wouldn't for blocks written in "partial flush"
2525 // mode, and it would block us indefinitely while trying to read another flate
2526 // block. The partial reader returns data earlier, but still eagerly consumes all
2527 // blocks in its buffer.
2528 // todo: also _write_ in partial mode since it uses fewer bytes than a sync flush (which needs an additional 4 bytes for the zero-length data block). we need a writer that can flush in partial mode first. writing with sync flush will work with clients that themselves write with partial flush.
2529 fr := flate.NewReaderPartial(rc)
2530 c.tr = moxio.NewTraceReader(c.log, "C: ", fr)
2531 c.br = bufio.NewReader(c.tr)
2534// STARTTLS enables TLS on the connection, after a plain text start.
2535// Only allowed if TLS isn't already enabled, either through connecting to a
2536// TLS-enabled TCP port, or a previous STARTTLS command.
2537// After STARTTLS, plain text authentication typically becomes available.
2539// Status: Not authenticated.
2540func (c *conn) cmdStarttls(tag, cmd string, p *parser) {
2549 if c.baseTLSConfig == nil {
2550 xsyntaxErrorf("starttls not announced")
2553 conn := xprefixConn(c.conn, c.br)
2554 // We add the cid to facilitate debugging in case of TLS connection failure.
2555 c.ok(tag, cmd+" ("+mox.ReceivedID(c.cid)+")")
2557 c.xtlsHandshakeAndAuthenticate(conn)
2560 // We are not sending unsolicited CAPABILITIES for newly available authentication
2561 // mechanisms, clients can't depend on us sending it and should ask it themselves.
2565// Authenticate using SASL. Supports multiple back and forths between client and
2566// server to finish authentication, unlike LOGIN which is just a single
2567// username/password.
2569// We may already have ambient TLS credentials that have not been activated.
2571// Status: Not authenticated.
2572func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
2576 // For many failed auth attempts, slow down verification attempts.
2577 if c.authFailed > 3 && authFailDelay > 0 {
2578 mox.Sleep(mox.Context, time.Duration(c.authFailed-3)*authFailDelay)
2581 // If authentication fails due to missing derived secrets, we don't hold it against
2582 // the connection. There is no way to indicate server support for an authentication
2583 // mechanism, but that a mechanism won't work for an account.
2584 var missingDerivedSecrets bool
2586 c.authFailed++ // Compensated on success.
2588 if missingDerivedSecrets {
2591 // On the 3rd failed authentication, start responding slowly. Successful auth will
2592 // cause fast responses again.
2593 if c.authFailed >= 3 {
2598 c.newLoginAttempt(true, "")
2600 if c.loginAttempt.Result == store.AuthSuccess {
2601 mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
2602 } else if !missingDerivedSecrets {
2603 mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
2609 authType := p.xatom()
2611 xreadInitial := func() []byte {
2615 line = c.xreadline(false)
2619 line = p.remainder()
2622 line = "" // Base64 decode will result in empty buffer.
2627 c.loginAttempt.Result = store.AuthAborted
2628 xsyntaxErrorf("authenticate aborted by client")
2630 buf, err := base64.StdEncoding.DecodeString(line)
2632 xsyntaxErrorf("parsing base64: %v", err)
2637 xreadContinuation := func() []byte {
2638 line := c.xreadline(false)
2640 c.loginAttempt.Result = store.AuthAborted
2641 xsyntaxErrorf("authenticate aborted by client")
2643 buf, err := base64.StdEncoding.DecodeString(line)
2645 xsyntaxErrorf("parsing base64: %v", err)
2650 // The various authentication mechanisms set account and username. We may already
2651 // have an account and username from TLS client authentication. Afterwards, we
2652 // check that the account is the same.
2653 var account *store.Account
2657 err := account.Close()
2658 c.xsanity(err, "close account")
2662 switch strings.ToUpper(authType) {
2664 c.loginAttempt.AuthMech = "plain"
2666 if !c.noRequireSTARTTLS && !c.tls {
2668 xusercodeErrorf("PRIVACYREQUIRED", "tls required for login")
2671 // Plain text passwords, mark as traceauth.
2672 defer c.xtraceread(mlog.LevelTraceauth)()
2673 buf := xreadInitial()
2674 c.xtraceread(mlog.LevelTrace) // Restore.
2675 plain := bytes.Split(buf, []byte{0})
2676 if len(plain) != 3 {
2677 xsyntaxErrorf("bad plain auth data, expected 3 nul-separated tokens, got %d tokens", len(plain))
2679 authz := norm.NFC.String(string(plain[0]))
2680 username = norm.NFC.String(string(plain[1]))
2681 password := string(plain[2])
2682 c.loginAttempt.LoginAddress = username
2684 if authz != "" && authz != username {
2685 xusercodeErrorf("AUTHORIZATIONFAILED", "cannot assume role")
2689 account, c.loginAttempt.AccountName, err = store.OpenEmailAuth(c.log, username, password, false)
2691 if errors.Is(err, store.ErrUnknownCredentials) {
2692 c.loginAttempt.Result = store.AuthBadCredentials
2693 c.log.Info("authentication failed", slog.String("username", username))
2694 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2696 xusercodeErrorf("", "error")
2700 c.loginAttempt.AuthMech = strings.ToLower(authType)
2706 chal := fmt.Sprintf("<%d.%d@%s>", uint64(mox.CryptoRandInt()), time.Now().UnixNano(), mox.Conf.Static.HostnameDomain.ASCII)
2707 c.xwritelinef("+ %s", base64.StdEncoding.EncodeToString([]byte(chal)))
2709 resp := xreadContinuation()
2710 t := strings.Split(string(resp), " ")
2711 if len(t) != 2 || len(t[1]) != 2*md5.Size {
2712 xsyntaxErrorf("malformed cram-md5 response")
2714 username = norm.NFC.String(t[0])
2715 c.loginAttempt.LoginAddress = username
2716 c.log.Debug("cram-md5 auth", slog.String("address", username))
2718 account, c.loginAttempt.AccountName, _, err = store.OpenEmail(c.log, username, false)
2720 if errors.Is(err, store.ErrUnknownCredentials) {
2721 c.loginAttempt.Result = store.AuthBadCredentials
2722 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2723 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2725 xserverErrorf("looking up address: %v", err)
2727 var ipadhash, opadhash hash.Hash
2728 account.WithRLock(func() {
2729 err := account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
2730 password, err := bstore.QueryTx[store.Password](tx).Get()
2731 if err == bstore.ErrAbsent {
2732 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2733 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2739 ipadhash = password.CRAMMD5.Ipad
2740 opadhash = password.CRAMMD5.Opad
2743 xcheckf(err, "tx read")
2745 if ipadhash == nil || opadhash == nil {
2746 c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", slog.String("username", username))
2747 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2748 missingDerivedSecrets = true
2749 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2753 ipadhash.Write([]byte(chal))
2754 opadhash.Write(ipadhash.Sum(nil))
2755 digest := fmt.Sprintf("%x", opadhash.Sum(nil))
2757 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2758 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2761 case "SCRAM-SHA-256-PLUS", "SCRAM-SHA-256", "SCRAM-SHA-1-PLUS", "SCRAM-SHA-1":
2762 // todo: improve handling of errors during scram. e.g. invalid parameters. should we abort the imap command, or continue until the end and respond with a scram-level error?
2763 // todo: use single implementation between ../imapserver/server.go and ../smtpserver/server.go
2765 // No plaintext credentials, we can log these normally.
2767 c.loginAttempt.AuthMech = strings.ToLower(authType)
2768 var h func() hash.Hash
2769 switch c.loginAttempt.AuthMech {
2770 case "scram-sha-1", "scram-sha-1-plus":
2772 case "scram-sha-256", "scram-sha-256-plus":
2775 xserverErrorf("missing case for scram variant")
2778 var cs *tls.ConnectionState
2779 requireChannelBinding := strings.HasSuffix(c.loginAttempt.AuthMech, "-plus")
2780 if requireChannelBinding && !c.tls {
2781 xuserErrorf("cannot use plus variant with tls channel binding without tls")
2784 xcs := c.conn.(*tls.Conn).ConnectionState()
2787 c0 := xreadInitial()
2788 ss, err := scram.NewServer(h, c0, cs, requireChannelBinding)
2790 c.log.Infox("scram protocol error", err, slog.Any("remote", c.remoteIP))
2791 xuserErrorf("scram protocol error: %s", err)
2793 username = ss.Authentication
2794 c.loginAttempt.LoginAddress = username
2795 c.log.Debug("scram auth", slog.String("authentication", username))
2796 // We check for login being disabled when finishing.
2797 account, c.loginAttempt.AccountName, _, err = store.OpenEmail(c.log, username, false)
2799 // todo: we could continue scram with a generated salt, deterministically generated
2800 // from the username. that way we don't have to store anything but attackers cannot
2801 // learn if an account exists. same for absent scram saltedpassword below.
2802 xuserErrorf("scram not possible")
2804 if ss.Authorization != "" && ss.Authorization != username {
2805 xuserErrorf("authentication with authorization for different user not supported")
2807 var xscram store.SCRAM
2808 account.WithRLock(func() {
2809 err := account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
2810 password, err := bstore.QueryTx[store.Password](tx).Get()
2811 if err == bstore.ErrAbsent {
2812 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2813 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2815 xcheckf(err, "fetching credentials")
2816 switch c.loginAttempt.AuthMech {
2817 case "scram-sha-1", "scram-sha-1-plus":
2818 xscram = password.SCRAMSHA1
2819 case "scram-sha-256", "scram-sha-256-plus":
2820 xscram = password.SCRAMSHA256
2822 xserverErrorf("missing case for scram credentials")
2824 if len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0 {
2825 missingDerivedSecrets = true
2826 c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", slog.String("username", username))
2827 xuserErrorf("scram not possible")
2831 xcheckf(err, "read tx")
2833 s1, err := ss.ServerFirst(xscram.Iterations, xscram.Salt)
2834 xcheckf(err, "scram first server step")
2835 c.xwritelinef("+ %s", base64.StdEncoding.EncodeToString([]byte(s1)))
2836 c2 := xreadContinuation()
2837 s3, err := ss.Finish(c2, xscram.SaltedPassword)
2839 c.xwritelinef("+ %s", base64.StdEncoding.EncodeToString([]byte(s3)))
2842 c.xreadline(false) // Should be "*" for cancellation.
2843 if errors.Is(err, scram.ErrInvalidProof) {
2844 c.loginAttempt.Result = store.AuthBadCredentials
2845 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2846 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
2847 } else if errors.Is(err, scram.ErrChannelBindingsDontMatch) {
2848 c.loginAttempt.Result = store.AuthBadChannelBinding
2849 c.log.Warn("bad channel binding during authentication, potential mitm", slog.String("username", username), slog.Any("remote", c.remoteIP))
2850 xusercodeErrorf("AUTHENTICATIONFAILED", "channel bindings do not match, potential mitm")
2851 } else if errors.Is(err, scram.ErrInvalidEncoding) {
2852 c.loginAttempt.Result = store.AuthBadProtocol
2853 c.log.Infox("bad scram protocol message", err, slog.String("username", username), slog.Any("remote", c.remoteIP))
2854 xuserErrorf("bad scram protocol message: %s", err)
2856 xuserErrorf("server final: %w", err)
2860 // The message should be empty. todo: should we require it is empty?
2864 c.loginAttempt.AuthMech = "external"
2867 buf := xreadInitial()
2868 username = norm.NFC.String(string(buf))
2869 c.loginAttempt.LoginAddress = username
2872 xusercodeErrorf("AUTHENTICATIONFAILED", "tls required for tls client certificate authentication")
2874 if c.account == nil {
2875 xusercodeErrorf("AUTHENTICATIONFAILED", "missing client certificate, required for tls client certificate authentication")
2879 username = c.username
2880 c.loginAttempt.LoginAddress = username
2883 account, c.loginAttempt.AccountName, _, err = store.OpenEmail(c.log, username, false)
2884 xcheckf(err, "looking up username from tls client authentication")
2887 c.loginAttempt.AuthMech = "(unrecognized)"
2888 xuserErrorf("method not supported")
2891 if accConf, ok := account.Conf(); !ok {
2892 xserverErrorf("cannot get account config")
2893 } else if accConf.LoginDisabled != "" {
2894 c.loginAttempt.Result = store.AuthLoginDisabled
2895 c.log.Info("account login disabled", slog.String("username", username))
2896 // No AUTHENTICATIONFAILED code, clients could prompt users for different password.
2897 xuserErrorf("%w: %s", store.ErrLoginDisabled, accConf.LoginDisabled)
2900 // We may already have TLS credentials. They won't have been enabled, or we could
2901 // get here due to the state machine that doesn't allow authentication while being
2902 // authenticated. But allow another SASL authentication, but it has to be for the
2903 // same account. It can be for a different username (email address) of the account.
2904 if c.account != nil {
2905 if account != c.account {
2906 c.log.Debug("sasl authentication for different account than tls client authentication, aborting connection",
2907 slog.String("saslmechanism", c.loginAttempt.AuthMech),
2908 slog.String("saslaccount", account.Name),
2909 slog.String("tlsaccount", c.account.Name),
2910 slog.String("saslusername", username),
2911 slog.String("tlsusername", c.username),
2913 xusercodeErrorf("AUTHENTICATIONFAILED", "authentication failed, tls client certificate public key belongs to another account")
2914 } else if username != c.username {
2915 c.log.Debug("sasl authentication for different username than tls client certificate authentication, switching to sasl username",
2916 slog.String("saslmechanism", c.loginAttempt.AuthMech),
2917 slog.String("saslusername", username),
2918 slog.String("tlsusername", c.username),
2919 slog.String("account", c.account.Name),
2924 account = nil // Prevent cleanup.
2926 c.username = username
2928 c.comm = store.RegisterComm(c.account)
2932 c.loginAttempt.AccountName = c.account.Name
2933 c.loginAttempt.LoginAddress = c.username
2934 c.loginAttempt.Result = store.AuthSuccess
2936 c.state = stateAuthenticated
2937 c.xwriteresultf("%s OK [CAPABILITY %s] authenticate done", tag, c.capabilities())
2940// Login logs in with username and password.
2942// Status: Not authenticated.
2943func (c *conn) cmdLogin(tag, cmd string, p *parser) {
2946 c.newLoginAttempt(true, "login")
2948 if c.loginAttempt.Result == store.AuthSuccess {
2949 mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
2951 mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
2955 // todo: get this line logged with traceauth. the plaintext password is included on the command line, which we've already read (before dispatching to this function).
2959 username := norm.NFC.String(p.xastring())
2960 c.loginAttempt.LoginAddress = username
2962 password := p.xastring()
2965 if !c.noRequireSTARTTLS && !c.tls {
2967 xusercodeErrorf("PRIVACYREQUIRED", "tls required for login")
2970 // For many failed auth attempts, slow down verification attempts.
2971 if c.authFailed > 3 && authFailDelay > 0 {
2972 mox.Sleep(mox.Context, time.Duration(c.authFailed-3)*authFailDelay)
2974 c.authFailed++ // Compensated on success.
2976 // On the 3rd failed authentication, start responding slowly. Successful auth will
2977 // cause fast responses again.
2978 if c.authFailed >= 3 {
2983 account, accName, err := store.OpenEmailAuth(c.log, username, password, true)
2984 c.loginAttempt.AccountName = accName
2987 if errors.Is(err, store.ErrUnknownCredentials) {
2988 c.loginAttempt.Result = store.AuthBadCredentials
2989 code = "AUTHENTICATIONFAILED"
2990 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2991 } else if errors.Is(err, store.ErrLoginDisabled) {
2992 c.loginAttempt.Result = store.AuthLoginDisabled
2993 c.log.Info("account login disabled", slog.String("username", username))
2994 // There is no specific code for "account disabled" in IMAP. AUTHORIZATIONFAILED is
2995 // not a good idea, it will prompt users for a password. ALERT seems reasonable,
2996 // but may cause email clients to suppress the message since we are not yet
2998 xuserErrorf("%s", err)
3000 xusercodeErrorf(code, "login failed")
3004 err := account.Close()
3005 c.xsanity(err, "close account")
3009 // We may already have TLS credentials. They won't have been enabled, or we could
3010 // get here due to the state machine that doesn't allow authentication while being
3011 // authenticated. But allow another SASL authentication, but it has to be for the
3012 // same account. It can be for a different username (email address) of the account.
3013 if c.account != nil {
3014 if account != c.account {
3015 c.log.Debug("sasl authentication for different account than tls client authentication, aborting connection",
3016 slog.String("saslmechanism", "login"),
3017 slog.String("saslaccount", account.Name),
3018 slog.String("tlsaccount", c.account.Name),
3019 slog.String("saslusername", username),
3020 slog.String("tlsusername", c.username),
3022 xusercodeErrorf("AUTHENTICATIONFAILED", "authentication failed, tls client certificate public key belongs to another account")
3023 } else if username != c.username {
3024 c.log.Debug("sasl authentication for different username than tls client certificate authentication, switching to sasl username",
3025 slog.String("saslmechanism", "login"),
3026 slog.String("saslusername", username),
3027 slog.String("tlsusername", c.username),
3028 slog.String("account", c.account.Name),
3033 account = nil // Prevent cleanup.
3035 c.username = username
3037 c.comm = store.RegisterComm(c.account)
3039 c.loginAttempt.LoginAddress = c.username
3040 c.loginAttempt.AccountName = c.account.Name
3041 c.loginAttempt.Result = store.AuthSuccess
3044 c.state = stateAuthenticated
3045 c.xwriteresultf("%s OK [CAPABILITY %s] login done", tag, c.capabilities())
3048// Enable explicitly opts in to an extension. A server can typically send new kinds
3049// of responses to a client. Most extensions do not require an ENABLE because a
3050// client implicitly opts in to new response syntax by making a requests that uses
3051// new optional extension request syntax.
3053// State: Authenticated and selected.
3054func (c *conn) cmdEnable(tag, cmd string, p *parser) {
3060 caps := []string{p.xatom()}
3063 caps = append(caps, p.xatom())
3066 // Clients should only send capabilities that need enabling.
3067 // We should only echo that we recognize as needing enabling.
3070 for _, s := range caps {
3071 cap := capability(strings.ToUpper(s))
3076 c.enabled[cap] = true
3079 c.enabled[cap] = true
3083 c.enabled[cap] = true
3086 c.enabled[cap] = true
3093 if qresync && !c.enabled[capCondstore] {
3094 c.xensureCondstore(nil)
3095 enabled += " CONDSTORE"
3099 c.xbwritelinef("* ENABLED%s", enabled)
3104// If a mailbox is selected, an untagged OK with HIGHESTMODSEQ is written to the
3105// client. If tx is non-nil, it is used to read the HIGHESTMODSEQ from the
3106// database. Otherwise a new read-only transaction is created.
3107func (c *conn) xensureCondstore(tx *bstore.Tx) {
3108 if !c.enabled[capCondstore] {
3109 c.enabled[capCondstore] = true
3110 // todo spec: can we send an untagged enabled response?
3112 if c.mailboxID <= 0 {
3116 var mb store.Mailbox
3118 c.xdbread(func(tx *bstore.Tx) {
3119 mb = c.xmailboxID(tx, c.mailboxID)
3122 mb = c.xmailboxID(tx, c.mailboxID)
3124 c.xbwritelinef("* OK [HIGHESTMODSEQ %d] after condstore-enabling command", mb.ModSeq.Client())
3128// State: Authenticated and selected.
3129func (c *conn) cmdSelect(tag, cmd string, p *parser) {
3130 c.cmdSelectExamine(true, tag, cmd, p)
3133// State: Authenticated and selected.
3134func (c *conn) cmdExamine(tag, cmd string, p *parser) {
3135 c.cmdSelectExamine(false, tag, cmd, p)
3138// Select and examine are almost the same commands. Select just opens a mailbox for
3139// read/write and examine opens a mailbox readonly.
3141// State: Authenticated and selected.
3142func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) {
3150 name := p.xmailbox()
3152 var qruidvalidity uint32
3153 var qrmodseq int64 // QRESYNC required parameters.
3154 var qrknownUIDs, qrknownSeqSet, qrknownUIDSet *numSet // QRESYNC optional parameters.
3156 seen := map[string]bool{}
3158 for len(seen) == 0 || !p.take(")") {
3159 w := p.xtakelist("CONDSTORE", "QRESYNC")
3161 xsyntaxErrorf("duplicate select parameter %s", w)
3171 // Note: unlike with CONDSTORE, there are no QRESYNC-related commands/parameters
3172 // that enable capabilities.
3173 if !c.enabled[capQresync] {
3175 xsyntaxErrorf("QRESYNC must first be enabled")
3181 qrmodseq = p.xnznumber64()
3183 seqMatchData := p.take("(")
3187 seqMatchData = p.take(" (")
3190 ss0 := p.xnumSet0(false, false)
3191 qrknownSeqSet = &ss0
3193 ss1 := p.xnumSet0(false, false)
3194 qrknownUIDSet = &ss1
3200 panic("missing case for select param " + w)
3206 // Deselect before attempting the new select. This means we will deselect when an
3207 // error occurs during select.
3209 if c.state == stateSelected {
3211 c.xbwritelinef("* OK [CLOSED] x")
3215 if c.uidonly && qrknownSeqSet != nil {
3217 xsyntaxCodeErrorf("UIDREQUIRED", "cannot use message sequence match data with uidonly enabled")
3220 name = xcheckmailboxname(name, true)
3222 var mb store.Mailbox
3223 c.account.WithRLock(func() {
3224 c.xdbread(func(tx *bstore.Tx) {
3225 mb = c.xmailbox(tx, name, "")
3227 var firstUnseen msgseq = 0
3229 c.uidnext = mb.UIDNext
3231 c.exists = uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
3233 c.uids = []store.UID{}
3235 q := bstore.QueryTx[store.Message](tx)
3236 q.FilterNonzero(store.Message{MailboxID: mb.ID})
3237 q.FilterEqual("Expunged", false)
3239 err := q.ForEach(func(m store.Message) error {
3240 c.uids = append(c.uids, m.UID)
3241 if firstUnseen == 0 && !m.Seen {
3242 firstUnseen = msgseq(len(c.uids))
3246 xcheckf(err, "fetching uids")
3248 c.exists = uint32(len(c.uids))
3252 if len(mb.Keywords) > 0 {
3253 flags = " " + strings.Join(mb.Keywords, " ")
3255 c.xbwritelinef(`* FLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent%s)`, flags)
3256 c.xbwritelinef(`* OK [PERMANENTFLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*)] x`)
3257 if !c.enabled[capIMAP4rev2] {
3258 c.xbwritelinef(`* 0 RECENT`)
3260 c.xbwritelinef(`* %d EXISTS`, c.exists)
3261 if !c.enabled[capIMAP4rev2] && firstUnseen > 0 {
3263 c.xbwritelinef(`* OK [UNSEEN %d] x`, firstUnseen)
3265 c.xbwritelinef(`* OK [UIDVALIDITY %d] x`, mb.UIDValidity)
3266 c.xbwritelinef(`* OK [UIDNEXT %d] x`, mb.UIDNext)
3267 c.xbwritelinef(`* LIST () "/" %s`, mailboxt(mb.Name).pack(c))
3268 if c.enabled[capCondstore] {
3271 c.xbwritelinef(`* OK [HIGHESTMODSEQ %d] x`, mb.ModSeq.Client())
3275 if qruidvalidity == mb.UIDValidity {
3276 // We send the vanished UIDs at the end, so we can easily combine the modseq
3277 // changes and vanished UIDs that result from that, with the vanished UIDs from the
3278 // case where we don't store enough history.
3279 vanishedUIDs := map[store.UID]struct{}{}
3281 var preVanished store.UID
3282 var oldClientUID store.UID
3283 // If samples of known msgseq and uid pairs are given (they must be in order), we
3284 // use them to determine the earliest UID for which we send VANISHED responses.
3286 if qrknownSeqSet != nil {
3287 if !qrknownSeqSet.isBasicIncreasing() {
3288 xuserErrorf("QRESYNC known message sequence set must be numeric and strictly increasing")
3290 if !qrknownUIDSet.isBasicIncreasing() {
3291 xuserErrorf("QRESYNC known uid set must be numeric and strictly increasing")
3293 seqiter := qrknownSeqSet.newIter()
3294 uiditer := qrknownUIDSet.newIter()
3296 msgseq, ok0 := seqiter.Next()
3297 uid, ok1 := uiditer.Next()
3300 } else if !ok0 || !ok1 {
3301 xsyntaxErrorf("invalid combination of known sequence set and uid set, must be of equal length")
3303 i := int(msgseq - 1)
3304 // Access to c.uids is safe, qrknownSeqSet and uidonly cannot both be set.
3305 if i < 0 || i >= int(c.exists) || c.uids[i] != store.UID(uid) {
3306 if uidSearch(c.uids, store.UID(uid)) <= 0 {
3307 // We will check this old client UID for consistency below.
3308 oldClientUID = store.UID(uid)
3312 preVanished = store.UID(uid + 1)
3316 // We gather vanished UIDs and report them at the end. This seems OK because we
3317 // already sent HIGHESTMODSEQ, and a client should know not to commit that value
3318 // until after it has seen the tagged OK of this command. The RFC has a remark
3319 // about ordering of some untagged responses, it's not immediately clear what it
3320 // means, but given the examples appears to allude to servers that decide to not
3321 // send expunge/vanished before the tagged OK.
3324 if oldClientUID > 0 {
3325 // The client sent a UID that is now removed. This is typically fine. But we check
3326 // that it is consistent with the modseq the client sent. If the UID already didn't
3327 // exist at that modseq, the client may be missing some information.
3328 q := bstore.QueryTx[store.Message](tx)
3329 q.FilterNonzero(store.Message{MailboxID: mb.ID, UID: oldClientUID})
3332 // If client claims to be up to date up to and including qrmodseq, and the message
3333 // was deleted at or before that time, we send changes from just before that
3334 // modseq, and we send vanished for all UIDs.
3335 if m.Expunged && qrmodseq >= m.ModSeq.Client() {
3336 qrmodseq = m.ModSeq.Client() - 1
3339 c.xbwritelinef("* OK [ALERT] Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full synchronization recommended.")
3341 } else if err != bstore.ErrAbsent {
3342 xcheckf(err, "checking old client uid")
3346 q := bstore.QueryTx[store.Message](tx)
3347 q.FilterNonzero(store.Message{MailboxID: mb.ID})
3348 // Note: we don't filter by Expunged.
3349 q.FilterGreater("ModSeq", store.ModSeqFromClient(qrmodseq))
3350 q.FilterLessEqual("ModSeq", mb.ModSeq)
3351 q.FilterLess("UID", c.uidnext)
3353 err := q.ForEach(func(m store.Message) error {
3354 if m.Expunged && m.UID < preVanished {
3358 if qrknownUIDs != nil && !qrknownUIDs.contains(uint32(m.UID)) {
3362 vanishedUIDs[m.UID] = struct{}{}
3367 c.xbwritelinef("* %d UIDFETCH (FLAGS %s MODSEQ (%d))", m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
3368 } else if msgseq := c.sequence(m.UID); msgseq > 0 {
3369 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s MODSEQ (%d))", msgseq, m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
3373 xcheckf(err, "listing changed messages")
3375 highDeletedModSeq, err := c.account.HighestDeletedModSeq(tx)
3376 xcheckf(err, "getting highest deleted modseq")
3378 // If we don't have enough history, we go through all UIDs and look them up, and
3379 // add them to the vanished list if they have disappeared.
3380 if qrmodseq < highDeletedModSeq.Client() {
3381 // If no "known uid set" was in the request, we substitute 1:max or the empty set.
3383 if qrknownUIDs == nil {
3384 qrknownUIDs = &numSet{ranges: []numRange{{first: setNumber{number: 1}, last: &setNumber{number: uint32(c.uidnext - 1)}}}}
3388 // note: qrknownUIDs will not contain "*".
3389 for _, r := range qrknownUIDs.xinterpretStar(func() store.UID { return 0 }).ranges {
3390 // Gather UIDs for this range.
3391 var uids []store.UID
3392 q := bstore.QueryTx[store.Message](tx)
3393 q.FilterNonzero(store.Message{MailboxID: mb.ID})
3394 q.FilterEqual("Expunged", false)
3396 q.FilterEqual("UID", r.first.number)
3398 q.FilterGreaterEqual("UID", r.first.number)
3399 q.FilterLessEqual("UID", r.last.number)
3402 for m, err := range q.All() {
3403 xcheckf(err, "enumerating uids")
3404 uids = append(uids, m.UID)
3407 // Find UIDs missing from the database.
3410 uid, ok := iter.Next()
3414 if uidSearch(uids, store.UID(uid)) <= 0 {
3415 vanishedUIDs[store.UID(uid)] = struct{}{}
3420 // Ensure it is in ascending order, no needless first/last ranges. qrknownUIDs cannot contain a star.
3421 iter := qrknownUIDs.newIter()
3423 v, ok := iter.Next()
3427 if c.sequence(store.UID(v)) <= 0 {
3428 vanishedUIDs[store.UID(v)] = struct{}{}
3434 // Now that we have all vanished UIDs, send them over compactly.
3435 if len(vanishedUIDs) > 0 {
3436 l := slices.Sorted(maps.Keys(vanishedUIDs))
3438 for _, s := range compactUIDSet(l).Strings(4*1024 - 32) {
3439 c.xbwritelinef("* VANISHED (EARLIER) %s", s)
3447 c.xbwriteresultf("%s OK [READ-WRITE] x", tag)
3450 c.xbwriteresultf("%s OK [READ-ONLY] x", tag)
3454 c.state = stateSelected
3455 c.searchResult = nil
3459// Create makes a new mailbox, and its parents too if absent.
3461// State: Authenticated and selected.
3462func (c *conn) cmdCreate(tag, cmd string, p *parser) {
3468 name := p.xmailbox()
3470 var useAttrs []string // Special-use attributes without leading \.
3473 // We only support "USE", and there don't appear to be more types of parameters.
3478 useAttrs = append(useAttrs, p.xatom())
3494 name = xcheckmailboxname(name, false)
3496 var specialUse store.SpecialUse
3497 specialUseBools := map[string]*bool{
3498 "archive": &specialUse.Archive,
3499 "drafts": &specialUse.Draft,
3500 "junk": &specialUse.Junk,
3501 "sent": &specialUse.Sent,
3502 "trash": &specialUse.Trash,
3504 for _, s := range useAttrs {
3505 p, ok := specialUseBools[strings.ToLower(s)]
3508 xusercodeErrorf("USEATTR", `cannot create mailbox with special-use attribute \%s`, s)
3513 var changes []store.Change
3514 var created []string // Created mailbox names.
3516 c.account.WithWLock(func() {
3517 c.xdbwrite(func(tx *bstore.Tx) {
3520 _, changes, created, exists, err = c.account.MailboxCreate(tx, name, specialUse)
3523 xuserErrorf("mailbox already exists")
3525 xcheckf(err, "creating mailbox")
3528 c.broadcast(changes)
3531 for _, n := range created {
3534 if c.enabled[capIMAP4rev2] && n == name && name != origName && !(name == "Inbox" || strings.HasPrefix(name, "Inbox/")) {
3535 oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, mailboxt(origName).pack(c))
3537 c.xbwritelinef(`* LIST (\Subscribed) "/" %s%s`, mailboxt(n).pack(c), oldname)
3542// Delete removes a mailbox and all its messages and annotations.
3543// Inbox cannot be removed.
3545// State: Authenticated and selected.
3546func (c *conn) cmdDelete(tag, cmd string, p *parser) {
3552 name := p.xmailbox()
3555 name = xcheckmailboxname(name, false)
3557 c.account.WithWLock(func() {
3558 var mb store.Mailbox
3559 var changes []store.Change
3561 c.xdbwrite(func(tx *bstore.Tx) {
3562 mb = c.xmailbox(tx, name, "NONEXISTENT")
3564 var hasChildren bool
3566 changes, hasChildren, err = c.account.MailboxDelete(context.TODO(), c.log, tx, &mb)
3568 xusercodeErrorf("HASCHILDREN", "mailbox has a child, only leaf mailboxes can be deleted")
3570 xcheckf(err, "deleting mailbox")
3573 c.broadcast(changes)
3579// Rename changes the name of a mailbox.
3580// Renaming INBOX is special, it moves the inbox messages to a new mailbox, leaving
3581// inbox empty, but copying metadata annotations.
3582// Renaming a mailbox with submailboxes also renames all submailboxes.
3583// Subscriptions stay with the old name, though newly created missing parent
3584// mailboxes for the destination name are automatically subscribed.
3586// State: Authenticated and selected.
3587func (c *conn) cmdRename(tag, cmd string, p *parser) {
3598 src = xcheckmailboxname(src, true)
3599 dst = xcheckmailboxname(dst, false)
3601 var cleanupIDs []int64
3603 for _, id := range cleanupIDs {
3604 p := c.account.MessagePath(id)
3606 c.xsanity(err, "cleaning up message")
3610 c.account.WithWLock(func() {
3611 var changes []store.Change
3613 c.xdbwrite(func(tx *bstore.Tx) {
3614 mbSrc := c.xmailbox(tx, src, "NONEXISTENT")
3616 // Handle common/simple case first.
3618 var modseq store.ModSeq
3619 var alreadyExists bool
3621 changes, _, alreadyExists, err = c.account.MailboxRename(tx, &mbSrc, dst, &modseq)
3623 xusercodeErrorf("ALREADYEXISTS", "%s", err)
3625 xcheckf(err, "renaming mailbox")
3629 // Inbox is very special. Unlike other mailboxes, its children are not moved. And
3630 // unlike a regular move, its messages are moved to a newly created mailbox. We do
3631 // indeed create a new destination mailbox and actually move the messages.
3633 exists, err := c.account.MailboxExists(tx, dst)
3634 xcheckf(err, "checking if destination mailbox exists")
3636 xusercodeErrorf("ALREADYEXISTS", "destination mailbox %q already exists", dst)
3639 xuserErrorf("cannot move inbox to itself")
3642 var modseq store.ModSeq
3643 mbDst, chl, err := c.account.MailboxEnsure(tx, dst, false, store.SpecialUse{}, &modseq)
3644 xcheckf(err, "creating destination mailbox")
3648 qa := bstore.QueryTx[store.Annotation](tx)
3649 qa.FilterNonzero(store.Annotation{MailboxID: mbSrc.ID})
3650 qa.FilterEqual("Expunged", false)
3651 annotations, err := qa.List()
3652 xcheckf(err, "get annotations to copy for inbox")
3653 for _, a := range annotations {
3655 a.MailboxID = mbDst.ID
3657 a.CreateSeq = modseq
3658 err := tx.Insert(&a)
3659 xcheckf(err, "copy annotation to destination mailbox")
3660 changes = append(changes, a.Change(mbDst.Name))
3662 c.xcheckMetadataSize(tx)
3664 // Build query that selects messages to move.
3665 q := bstore.QueryTx[store.Message](tx)
3666 q.FilterNonzero(store.Message{MailboxID: mbSrc.ID})
3667 q.FilterEqual("Expunged", false)
3670 newIDs, chl := c.xmoveMessages(tx, q, 0, modseq, &mbSrc, &mbDst)
3671 changes = append(changes, chl...)
3677 c.broadcast(changes)
3683// Subscribe marks a mailbox path as subscribed. The mailbox does not have to
3684// exist. Subscribed may mean an email client will show the mailbox in its UI
3685// and/or periodically fetch new messages for the mailbox.
3687// State: Authenticated and selected.
3688func (c *conn) cmdSubscribe(tag, cmd string, p *parser) {
3694 name := p.xmailbox()
3697 name = xcheckmailboxname(name, true)
3699 c.account.WithWLock(func() {
3700 var changes []store.Change
3702 c.xdbwrite(func(tx *bstore.Tx) {
3704 changes, err = c.account.SubscriptionEnsure(tx, name)
3705 xcheckf(err, "ensuring subscription")
3708 c.broadcast(changes)
3714// Unsubscribe marks a mailbox as not subscribed. The mailbox doesn't have to exist.
3716// State: Authenticated and selected.
3717func (c *conn) cmdUnsubscribe(tag, cmd string, p *parser) {
3723 name := p.xmailbox()
3726 name = xcheckmailboxname(name, true)
3728 c.account.WithWLock(func() {
3729 var changes []store.Change
3731 c.xdbwrite(func(tx *bstore.Tx) {
3733 err := tx.Delete(&store.Subscription{Name: name})
3734 if err == bstore.ErrAbsent {
3735 exists, err := c.account.MailboxExists(tx, name)
3736 xcheckf(err, "checking if mailbox exists")
3738 xuserErrorf("mailbox does not exist")
3742 xcheckf(err, "removing subscription")
3745 exists, err := c.account.MailboxExists(tx, name)
3746 xcheckf(err, "looking up mailbox existence")
3748 flags = []string{`\NonExistent`}
3751 changes = []store.Change{store.ChangeRemoveSubscription{MailboxName: name, ListFlags: flags}}
3754 c.broadcast(changes)
3756 // todo: can we send untagged message about a mailbox no longer being subscribed?
3762// LSUB command for listing subscribed mailboxes.
3763// Removed in IMAP4rev2, only in IMAP4rev1.
3765// State: Authenticated and selected.
3766func (c *conn) cmdLsub(tag, cmd string, p *parser) {
3774 pattern := p.xlistMailbox()
3777 re := xmailboxPatternMatcher(ref, []string{pattern})
3780 c.xdbread(func(tx *bstore.Tx) {
3781 q := bstore.QueryTx[store.Subscription](tx)
3783 subscriptions, err := q.List()
3784 xcheckf(err, "querying subscriptions")
3786 have := map[string]bool{}
3787 subscribedKids := map[string]bool{}
3788 ispercent := strings.HasSuffix(pattern, "%")
3789 for _, sub := range subscriptions {
3792 for p := mox.ParentMailboxName(name); p != ""; p = mox.ParentMailboxName(p) {
3793 subscribedKids[p] = true
3796 if !re.MatchString(name) {
3800 line := fmt.Sprintf(`* LSUB () "/" %s`, mailboxt(name).pack(c))
3801 lines = append(lines, line)
3809 qmb := bstore.QueryTx[store.Mailbox](tx)
3810 qmb.FilterEqual("Expunged", false)
3812 err = qmb.ForEach(func(mb store.Mailbox) error {
3813 if have[mb.Name] || !subscribedKids[mb.Name] || !re.MatchString(mb.Name) {
3816 line := fmt.Sprintf(`* LSUB (\NoSelect) "/" %s`, mailboxt(mb.Name).pack(c))
3817 lines = append(lines, line)
3820 xcheckf(err, "querying mailboxes")
3824 for _, line := range lines {
3825 c.xbwritelinef("%s", line)
3830// The namespace command returns the mailbox path separator. We only implement
3831// the personal mailbox hierarchy, no shared/other.
3833// In IMAP4rev2, it was an extension before.
3835// State: Authenticated and selected.
3836func (c *conn) cmdNamespace(tag, cmd string, p *parser) {
3843 c.xbwritelinef(`* NAMESPACE (("" "/")) NIL NIL`)
3847// The status command returns information about a mailbox, such as the number of
3848// messages, "uid validity", etc. Nowadays, the extended LIST command can return
3849// the same information about many mailboxes for one command.
3851// State: Authenticated and selected.
3852func (c *conn) cmdStatus(tag, cmd string, p *parser) {
3858 name := p.xmailbox()
3861 attrs := []string{p.xstatusAtt()}
3864 attrs = append(attrs, p.xstatusAtt())
3868 name = xcheckmailboxname(name, true)
3870 var mb store.Mailbox
3872 var responseLine string
3873 c.account.WithRLock(func() {
3874 c.xdbread(func(tx *bstore.Tx) {
3875 mb = c.xmailbox(tx, name, "")
3876 responseLine = c.xstatusLine(tx, mb, attrs)
3880 c.xbwritelinef("%s", responseLine)
3885func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) string {
3886 status := []string{}
3887 for _, a := range attrs {
3888 A := strings.ToUpper(a)
3891 status = append(status, A, fmt.Sprintf("%d", mb.Total+mb.Deleted))
3893 status = append(status, A, fmt.Sprintf("%d", mb.UIDNext))
3895 status = append(status, A, fmt.Sprintf("%d", mb.UIDValidity))
3897 status = append(status, A, fmt.Sprintf("%d", mb.Unseen))
3899 status = append(status, A, fmt.Sprintf("%d", mb.Deleted))
3901 status = append(status, A, fmt.Sprintf("%d", mb.Size))
3903 status = append(status, A, "0")
3906 status = append(status, A, "NIL")
3907 case "HIGHESTMODSEQ":
3909 status = append(status, A, fmt.Sprintf("%d", mb.ModSeq.Client()))
3910 case "DELETED-STORAGE":
3912 // How much storage space could be reclaimed by expunging messages with the
3913 // \Deleted flag. We could keep track of this number and return it efficiently.
3914 // Calculating it each time can be slow, and we don't know if clients request it.
3915 // Clients are not likely to set the deleted flag without immediately expunging
3916 // nowadays. Let's wait for something to need it to go through the trouble, and
3917 // always return 0 for now.
3918 status = append(status, A, "0")
3920 xsyntaxErrorf("unknown attribute %q", a)
3923 return fmt.Sprintf("* STATUS %s (%s)", mailboxt(mb.Name).pack(c), strings.Join(status, " "))
3926func flaglist(fl store.Flags, keywords []string) listspace {
3928 flag := func(v bool, s string) {
3930 l = append(l, bare(s))
3933 flag(fl.Seen, `\Seen`)
3934 flag(fl.Answered, `\Answered`)
3935 flag(fl.Flagged, `\Flagged`)
3936 flag(fl.Deleted, `\Deleted`)
3937 flag(fl.Draft, `\Draft`)
3938 flag(fl.Forwarded, `$Forwarded`)
3939 flag(fl.Junk, `$Junk`)
3940 flag(fl.Notjunk, `$NotJunk`)
3941 flag(fl.Phishing, `$Phishing`)
3942 flag(fl.MDNSent, `$MDNSent`)
3943 for _, k := range keywords {
3944 l = append(l, bare(k))
3949// Append adds a message to a mailbox.
3950// The MULTIAPPEND extension is implemented, allowing multiple flags/datetime/data
3953// State: Authenticated and selected.
3954func (c *conn) cmdAppend(tag, cmd string, p *parser) {
3958 // A message that we've (partially) read from the client, and will be delivering to
3960 type appendMsg struct {
3961 storeFlags store.Flags
3965 file *os.File // Message file we are appending. Can be nil if we are writing to a nopWriteCloser due to being over quota.
3968 m store.Message // New message. Delivered file for m.ID is removed on error.
3971 var appends []*appendMsg
3974 for _, a := range appends {
3975 if !commit && a.m.ID != 0 {
3976 p := c.account.MessagePath(a.m.ID)
3978 c.xsanity(err, "cleaning up temporary append file after error")
3985 name := p.xmailbox()
3988 // Check how much quota space is available. We'll keep track of remaining quota as
3989 // we accept multiple messages.
3990 quotaMsgMax := c.account.QuotaMessageSize()
3991 quotaUnlimited := quotaMsgMax == 0
3992 var quotaAvail int64
3994 if !quotaUnlimited {
3995 c.account.WithRLock(func() {
3996 c.xdbread(func(tx *bstore.Tx) {
3997 du := store.DiskUsage{ID: 1}
3999 xcheckf(err, "get quota disk usage")
4000 quotaAvail = quotaMsgMax - du.MessageSize
4005 var overQuota bool // For response code.
4006 var cancel bool // In case we've seen zero-sized message append.
4009 // Append msg early, for potential cleanup.
4011 appends = append(appends, &a)
4013 if p.hasPrefix("(") {
4014 // Error must be a syntax error, to properly abort the connection due to literal.
4016 a.storeFlags, a.keywords, err = store.ParseFlagsKeywords(p.xflagList())
4018 xsyntaxErrorf("parsing flags: %v", err)
4022 if p.hasPrefix(`"`) {
4023 a.time = p.xdateTime()
4028 // todo: only with utf8 should we we accept message headers with utf-8. we currently always accept them.
4029 // todo: this is only relevant if we also support the CATENATE extension?
4031 utf8 := p.take("UTF8 (")
4036 // For utf8, we already consumed the required ~ above.
4037 size, synclit := p.xliteralSize(!utf8, false)
4039 if !quotaUnlimited && !overQuota {
4041 overQuota = quotaAvail < 0
4049 // Check for mailbox on first iteration.
4050 if len(appends) <= 1 {
4051 name = xcheckmailboxname(name, true)
4052 c.xdbread(func(tx *bstore.Tx) {
4053 c.xmailbox(tx, name, "TRYCREATE")
4059 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", quotaMsgMax)
4064 xuserErrorf("empty message, cancelling append")
4067 // Read the message into a temporary file.
4069 a.file, err = store.CreateMessageTemp(c.log, "imap-append")
4070 xcheckf(err, "creating temp file for message")
4071 defer store.CloseRemoveTempFile(c.log, a.file, "temporary message file")
4076 // We'll discard the message and return an error as soon as we can (possible
4077 // synchronizing literal of next message, or after we've seen all messages).
4078 if overQuota || cancel {
4082 a.file, err = store.CreateMessageTemp(c.log, "imap-append")
4083 xcheckf(err, "creating temp file for message")
4084 defer store.CloseRemoveTempFile(c.log, a.file, "temporary message file")
4089 defer c.xtracewrite(mlog.LevelTracedata)()
4090 a.mw = message.NewWriter(f)
4091 msize, err := io.Copy(a.mw, io.LimitReader(c.br, size))
4092 c.xtracewrite(mlog.LevelTrace) // Restore.
4094 // Cannot use xcheckf due to %w handling of errIO.
4095 c.xbrokenf("reading literal message: %s (%w)", err, errIO)
4098 c.xbrokenf("read %d bytes for message, expected %d (%w)", msize, size, errIO)
4102 line := c.xreadline(false)
4103 p = newParser(line, c)
4108 // The MULTIAPPEND extension allows more appends.
4115 name = xcheckmailboxname(name, true)
4119 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", quotaMsgMax)
4124 xuserErrorf("empty message, cancelling append")
4127 var mb store.Mailbox
4129 var pendingChanges []store.Change
4131 // In case of panic.
4132 c.flushChanges(pendingChanges)
4137 c.account.WithWLock(func() {
4138 var changes []store.Change
4140 c.xdbwrite(func(tx *bstore.Tx) {
4141 mb = c.xmailbox(tx, name, "TRYCREATE")
4143 nkeywords := len(mb.Keywords)
4145 // Check quota for all messages at once.
4146 ok, maxSize, err := c.account.CanAddMessageSize(tx, totalSize)
4147 xcheckf(err, "checking quota")
4150 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
4153 modseq, err := c.account.NextModSeq(tx)
4154 xcheckf(err, "get next mod seq")
4158 msgDirs := map[string]struct{}{}
4159 for _, a := range appends {
4160 a.m = store.Message{
4162 MailboxOrigID: mb.ID,
4164 Flags: a.storeFlags,
4165 Keywords: a.keywords,
4171 // todo: do a single junk training
4172 err = c.account.MessageAdd(c.log, tx, &mb, &a.m, a.file, store.AddOpts{SkipDirSync: true})
4173 xcheckf(err, "delivering message")
4175 changes = append(changes, a.m.ChangeAddUID(mb))
4177 msgDirs[filepath.Dir(c.account.MessagePath(a.m.ID))] = struct{}{}
4180 changes = append(changes, mb.ChangeCounts())
4181 if nkeywords != len(mb.Keywords) {
4182 changes = append(changes, mb.ChangeKeywords())
4185 err = tx.Update(&mb)
4186 xcheckf(err, "updating mailbox counts")
4188 for dir := range msgDirs {
4189 err := moxio.SyncDir(c.log, dir)
4190 xcheckf(err, "sync dir")
4196 // Fetch pending changes, possibly with new UIDs, so we can apply them before adding our own new UID.
4197 overflow, pendingChanges = c.comm.Get()
4199 // Broadcast the change to other connections.
4200 c.broadcast(changes)
4203 if c.mailboxID == mb.ID {
4205 pendingChanges = nil
4206 c.xapplyChanges(overflow, l, true)
4207 for _, a := range appends {
4208 c.uidAppend(a.m.UID)
4210 // todo spec: with condstore/qresync, is there a mechanism to let the client know the modseq for the appended uid? in theory an untagged fetch with the modseq after the OK APPENDUID could make sense, but this probably isn't allowed.
4211 c.xbwritelinef("* %d EXISTS", c.exists)
4217 if len(appends) == 1 {
4218 uidset = fmt.Sprintf("%d", appends[0].m.UID)
4220 uidset = fmt.Sprintf("%d:%d", appends[0].m.UID, appends[len(appends)-1].m.UID)
4222 c.xwriteresultf("%s OK [APPENDUID %d %s] appended", tag, mb.UIDValidity, uidset)
4225// Idle makes a client wait until the server sends untagged updates, e.g. about
4226// message delivery or mailbox create/rename/delete/subscription, etc. It allows a
4227// client to get updates in real-time, not needing the use for NOOP.
4229// State: Authenticated and selected.
4230func (c *conn) cmdIdle(tag, cmd string, p *parser) {
4237 c.xwritelinef("+ waiting")
4239 // With NOTIFY enabled, flush all pending changes.
4240 if c.notify != nil && len(c.notify.Delayed) > 0 {
4241 c.xapplyChanges(false, nil, true)
4249 case le := <-c.lineChan():
4251 if err := le.err; err != nil {
4252 if errors.Is(le.err, os.ErrDeadlineExceeded) {
4253 err := c.conn.SetDeadline(time.Now().Add(10 * time.Second))
4254 c.log.Check(err, "setting deadline")
4255 c.xwritelinef("* BYE inactive")
4258 if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) {
4259 c.xbrokenf("%s (%w)", err, errIO)
4265 case <-c.comm.Pending:
4266 overflow, changes := c.comm.Get()
4267 c.xapplyChanges(overflow, changes, true)
4269 case <-mox.Shutdown.Done():
4271 c.xwritelinef("* BYE shutting down")
4272 c.xbrokenf("shutting down (%w)", errIO)
4276 // Reset the write deadline. In case of little activity, with a command timeout of
4277 // 30 minutes, we have likely passed it.
4278 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
4279 c.log.Check(err, "setting write deadline")
4281 if strings.ToUpper(line) != "DONE" {
4282 // We just close the connection because our protocols are out of sync.
4283 c.xbrokenf("%w: in IDLE, expected DONE", errIO)
4289// Return the quota root for a mailbox name and any current quota's.
4291// State: Authenticated and selected.
4292func (c *conn) cmdGetquotaroot(tag, cmd string, p *parser) {
4297 name := p.xmailbox()
4300 // This mailbox does not have to exist. Caller just wants to know which limits
4301 // would apply. We only have one limit, so we don't use the name otherwise.
4303 name = xcheckmailboxname(name, true)
4305 // Get current usage for account.
4306 var quota, size int64 // Account only has a quota if > 0.
4307 c.account.WithRLock(func() {
4308 quota = c.account.QuotaMessageSize()
4310 c.xdbread(func(tx *bstore.Tx) {
4311 du := store.DiskUsage{ID: 1}
4313 xcheckf(err, "gather used quota")
4314 size = du.MessageSize
4319 // We only have one per account quota, we name it "" like the examples in the RFC.
4321 c.xbwritelinef(`* QUOTAROOT %s ""`, astring(name).pack(c))
4323 // We only write the quota response if there is a limit. The syntax doesn't allow
4324 // an empty list, so we cannot send the current disk usage if there is no limit.
4327 c.xbwritelinef(`* QUOTA "" (STORAGE %d %d)`, (size+1024-1)/1024, (quota+1024-1)/1024)
4332// Return the quota for a quota root.
4334// State: Authenticated and selected.
4335func (c *conn) cmdGetquota(tag, cmd string, p *parser) {
4340 root := p.xastring()
4343 // We only have a per-account root called "".
4345 xuserErrorf("unknown quota root")
4348 var quota, size int64
4349 c.account.WithRLock(func() {
4350 quota = c.account.QuotaMessageSize()
4352 c.xdbread(func(tx *bstore.Tx) {
4353 du := store.DiskUsage{ID: 1}
4355 xcheckf(err, "gather used quota")
4356 size = du.MessageSize
4361 // We only write the quota response if there is a limit. The syntax doesn't allow
4362 // an empty list, so we cannot send the current disk usage if there is no limit.
4365 c.xbwritelinef(`* QUOTA "" (STORAGE %d %d)`, (size+1024-1)/1024, (quota+1024-1)/1024)
4370// Check is an old deprecated command that is supposed to execute some mailbox consistency checks.
4373func (c *conn) cmdCheck(tag, cmd string, p *parser) {
4379 c.account.WithRLock(func() {
4380 c.xdbread(func(tx *bstore.Tx) {
4381 c.xmailboxID(tx, c.mailboxID) // Validate.
4388// Close undoes select/examine, closing the currently opened mailbox and deleting
4389// messages that were marked for deletion with the \Deleted flag.
4392func (c *conn) cmdClose(tag, cmd string, p *parser) {
4399 c.xexpunge(nil, true)
4405// expunge messages marked for deletion in currently selected/active mailbox.
4406// if uidSet is not nil, only messages matching the set are expunged.
4408// Messages that have been marked expunged from the database are returned. While
4409// other sessions still reference the message, it is not cleared from the database
4410// yet, and the message file is not yet removed.
4412// The highest modseq in the mailbox is returned, typically associated with the
4413// removal of the messages, but if no messages were expunged the current latest max
4414// modseq for the mailbox is returned.
4415func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (expunged []store.Message, highestModSeq store.ModSeq) {
4416 c.account.WithWLock(func() {
4417 var changes []store.Change
4419 c.xdbwrite(func(tx *bstore.Tx) {
4420 mb, err := store.MailboxID(tx, c.mailboxID)
4421 if err == bstore.ErrAbsent || err == store.ErrMailboxExpunged {
4422 if missingMailboxOK {
4426 xusercodeErrorf("NONEXISTENT", "%w", store.ErrUnknownMailbox)
4428 xcheckf(err, "get mailbox")
4430 xlastUID := c.newCachedLastUID(tx, c.mailboxID, func(err error) { xuserErrorf("%s", err) })
4432 qm := bstore.QueryTx[store.Message](tx)
4433 qm.FilterNonzero(store.Message{MailboxID: c.mailboxID})
4434 qm.FilterEqual("Deleted", true)
4435 qm.FilterEqual("Expunged", false)
4436 qm.FilterLess("UID", c.uidnext)
4437 qm.FilterFn(func(m store.Message) bool {
4438 // Only remove if this session knows about the message and if present in optional
4440 return uidSet == nil || uidSet.xcontainsKnownUID(m.UID, c.searchResult, xlastUID)
4443 expunged, err = qm.List()
4444 xcheckf(err, "listing messages to expunge")
4446 if len(expunged) == 0 {
4447 highestModSeq = mb.ModSeq
4451 // Assign new modseq.
4452 modseq, err := c.account.NextModSeq(tx)
4453 xcheckf(err, "assigning next modseq")
4454 highestModSeq = modseq
4457 chremuids, chmbcounts, err := c.account.MessageRemove(c.log, tx, modseq, &mb, store.RemoveOpts{}, expunged...)
4458 xcheckf(err, "expunging messages")
4459 changes = append(changes, chremuids, chmbcounts)
4461 err = tx.Update(&mb)
4462 xcheckf(err, "update mailbox")
4465 c.broadcast(changes)
4468 return expunged, highestModSeq
4471// Unselect is similar to close in that it closes the currently active mailbox, but
4472// it does not remove messages marked for deletion.
4475func (c *conn) cmdUnselect(tag, cmd string, p *parser) {
4485// Expunge deletes messages marked with \Deleted in the currently selected mailbox.
4486// Clients are wiser to use UID EXPUNGE because it allows a UID sequence set to
4487// explicitly opt in to removing specific messages.
4490func (c *conn) cmdExpunge(tag, cmd string, p *parser) {
4497 xuserErrorf("mailbox open in read-only mode")
4500 c.cmdxExpunge(tag, cmd, nil)
4503// UID expunge deletes messages marked with \Deleted in the currently selected
4504// mailbox if they match a UID sequence set.
4507func (c *conn) cmdUIDExpunge(tag, cmd string, p *parser) {
4512 uidSet := p.xnumSet()
4516 xuserErrorf("mailbox open in read-only mode")
4519 c.cmdxExpunge(tag, cmd, &uidSet)
4522// Permanently delete messages for the currently selected/active mailbox. If uidset
4523// is not nil, only those UIDs are expunged.
4525func (c *conn) cmdxExpunge(tag, cmd string, uidSet *numSet) {
4528 expunged, highestModSeq := c.xexpunge(uidSet, false)
4531 var vanishedUIDs numSet
4532 qresync := c.enabled[capQresync]
4533 for _, m := range expunged {
4537 vanishedUIDs.append(uint32(m.UID))
4540 seq := c.xsequence(m.UID)
4541 c.sequenceRemove(seq, m.UID)
4543 vanishedUIDs.append(uint32(m.UID))
4545 c.xbwritelinef("* %d EXPUNGE", seq)
4548 if !vanishedUIDs.empty() {
4550 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
4551 c.xbwritelinef("* VANISHED %s", s)
4555 if c.enabled[capCondstore] {
4556 c.xwriteresultf("%s OK [HIGHESTMODSEQ %d] expunged", tag, highestModSeq.Client())
4563func (c *conn) cmdSearch(tag, cmd string, p *parser) {
4564 c.cmdxSearch(false, false, tag, cmd, p)
4568func (c *conn) cmdUIDSearch(tag, cmd string, p *parser) {
4569 c.cmdxSearch(true, false, tag, cmd, p)
4573func (c *conn) cmdFetch(tag, cmd string, p *parser) {
4574 c.cmdxFetch(false, tag, cmd, p)
4578func (c *conn) cmdUIDFetch(tag, cmd string, p *parser) {
4579 c.cmdxFetch(true, tag, cmd, p)
4583func (c *conn) cmdStore(tag, cmd string, p *parser) {
4584 c.cmdxStore(false, tag, cmd, p)
4588func (c *conn) cmdUIDStore(tag, cmd string, p *parser) {
4589 c.cmdxStore(true, tag, cmd, p)
4593func (c *conn) cmdCopy(tag, cmd string, p *parser) {
4594 c.cmdxCopy(false, tag, cmd, p)
4598func (c *conn) cmdUIDCopy(tag, cmd string, p *parser) {
4599 c.cmdxCopy(true, tag, cmd, p)
4603func (c *conn) cmdMove(tag, cmd string, p *parser) {
4604 c.cmdxMove(false, tag, cmd, p)
4608func (c *conn) cmdUIDMove(tag, cmd string, p *parser) {
4609 c.cmdxMove(true, tag, cmd, p)
4613func (c *conn) cmdReplace(tag, cmd string, p *parser) {
4614 c.cmdxReplace(false, tag, cmd, p)
4618func (c *conn) cmdUIDReplace(tag, cmd string, p *parser) {
4619 c.cmdxReplace(true, tag, cmd, p)
4622func (c *conn) gatherCopyMoveUIDs(tx *bstore.Tx, isUID bool, nums numSet) []store.UID {
4623 // Gather uids, then sort so we can return a consistently simple and hard to
4624 // misinterpret COPYUID/MOVEUID response. It seems safer to have UIDs in ascending
4625 // order, because requested uid set of 12:10 is equal to 10:12, so if we would just
4626 // echo whatever the client sends us without reordering, the client can reorder our
4627 // response and interpret it differently than we intended.
4629 return c.xnumSetEval(tx, isUID, nums)
4632// Copy copies messages from the currently selected/active mailbox to another named
4636func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
4643 name := p.xmailbox()
4646 name = xcheckmailboxname(name, true)
4648 // Files that were created during the copy. Remove them if the operation fails.
4651 for _, id := range newIDs {
4652 p := c.account.MessagePath(id)
4654 c.xsanity(err, "cleaning up created file")
4659 var uids []store.UID
4661 var mbDst store.Mailbox
4663 var newUIDs []store.UID
4664 var flags []store.Flags
4665 var keywords [][]string
4666 var modseq store.ModSeq // For messages in new mailbox, assigned when first message is copied.
4668 c.account.WithWLock(func() {
4670 c.xdbwrite(func(tx *bstore.Tx) {
4671 mbSrc := c.xmailboxID(tx, c.mailboxID) // Validate.
4673 mbDst = c.xmailbox(tx, name, "TRYCREATE")
4674 if mbDst.ID == mbSrc.ID {
4675 xuserErrorf("cannot copy to currently selected mailbox")
4678 uids = c.gatherCopyMoveUIDs(tx, isUID, nums)
4681 xuserErrorf("no matching messages to copy")
4684 nkeywords = len(mbDst.Keywords)
4687 modseq, err = c.account.NextModSeq(tx)
4688 xcheckf(err, "assigning next modseq")
4689 mbSrc.ModSeq = modseq
4690 mbDst.ModSeq = modseq
4692 err = tx.Update(&mbSrc)
4693 xcheckf(err, "updating source mailbox for modseq")
4695 // Reserve the uids in the destination mailbox.
4696 uidFirst := mbDst.UIDNext
4697 err = mbDst.UIDNextAdd(len(uids))
4698 xcheckf(err, "adding uid")
4700 // Fetch messages from database.
4701 q := bstore.QueryTx[store.Message](tx)
4702 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
4703 q.FilterEqual("UID", slicesAny(uids)...)
4704 q.FilterEqual("Expunged", false)
4705 xmsgs, err := q.List()
4706 xcheckf(err, "fetching messages")
4708 if len(xmsgs) != len(uids) {
4709 xserverErrorf("uid and message mismatch")
4712 // See if quota allows copy.
4714 for _, m := range xmsgs {
4717 if ok, maxSize, err := c.account.CanAddMessageSize(tx, totalSize); err != nil {
4718 xcheckf(err, "checking quota")
4721 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
4723 err = c.account.AddMessageSize(c.log, tx, totalSize)
4724 xcheckf(err, "updating disk usage")
4726 msgs := map[store.UID]store.Message{}
4727 for _, m := range xmsgs {
4730 nmsgs := make([]store.Message, len(xmsgs))
4732 conf, _ := c.account.Conf()
4734 mbKeywords := map[string]struct{}{}
4737 // Insert new messages into database.
4738 var origMsgIDs, newMsgIDs []int64
4739 for i, uid := range uids {
4742 xuserErrorf("messages changed, could not fetch requested uid")
4745 origMsgIDs = append(origMsgIDs, origID)
4747 m.UID = uidFirst + store.UID(i)
4748 m.CreateSeq = modseq
4750 m.MailboxID = mbDst.ID
4751 if m.IsReject && m.MailboxDestinedID != 0 {
4752 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
4753 // is used for reputation calculation during future deliveries.
4754 m.MailboxOrigID = m.MailboxDestinedID
4758 m.JunkFlagsForMailbox(mbDst, conf)
4760 err := tx.Insert(&m)
4761 xcheckf(err, "inserting message")
4764 newUIDs = append(newUIDs, m.UID)
4765 newMsgIDs = append(newMsgIDs, m.ID)
4766 flags = append(flags, m.Flags)
4767 keywords = append(keywords, m.Keywords)
4768 for _, kw := range m.Keywords {
4769 mbKeywords[kw] = struct{}{}
4772 qmr := bstore.QueryTx[store.Recipient](tx)
4773 qmr.FilterNonzero(store.Recipient{MessageID: origID})
4774 mrs, err := qmr.List()
4775 xcheckf(err, "listing message recipients")
4776 for _, mr := range mrs {
4779 err := tx.Insert(&mr)
4780 xcheckf(err, "inserting message recipient")
4783 mbDst.Add(m.MailboxCounts())
4786 mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, slices.Sorted(maps.Keys(mbKeywords)))
4788 err = tx.Update(&mbDst)
4789 xcheckf(err, "updating destination mailbox for uids, keywords and counts")
4791 // Copy message files to new message ID's.
4792 syncDirs := map[string]struct{}{}
4793 for i := range origMsgIDs {
4794 src := c.account.MessagePath(origMsgIDs[i])
4795 dst := c.account.MessagePath(newMsgIDs[i])
4796 dstdir := filepath.Dir(dst)
4797 if _, ok := syncDirs[dstdir]; !ok {
4798 os.MkdirAll(dstdir, 0770)
4799 syncDirs[dstdir] = struct{}{}
4801 err := moxio.LinkOrCopy(c.log, dst, src, nil, true)
4802 xcheckf(err, "link or copy file %q to %q", src, dst)
4803 newIDs = append(newIDs, newMsgIDs[i])
4806 for dir := range syncDirs {
4807 err := moxio.SyncDir(c.log, dir)
4808 xcheckf(err, "sync directory")
4811 err = c.account.RetrainMessages(context.TODO(), c.log, tx, nmsgs)
4812 xcheckf(err, "train copied messages")
4817 // Broadcast changes to other connections.
4818 if len(newUIDs) > 0 {
4819 changes := make([]store.Change, 0, len(newUIDs)+2)
4820 for i, uid := range newUIDs {
4821 add := store.ChangeAddUID{
4822 MailboxID: mbDst.ID,
4826 Keywords: keywords[i],
4827 MessageCountIMAP: mbDst.MessageCountIMAP(),
4828 Unseen: uint32(mbDst.MailboxCounts.Unseen),
4830 changes = append(changes, add)
4832 changes = append(changes, mbDst.ChangeCounts())
4833 if nkeywords != len(mbDst.Keywords) {
4834 changes = append(changes, mbDst.ChangeKeywords())
4836 c.broadcast(changes)
4841 c.xwriteresultf("%s OK [COPYUID %d %s %s] copied", tag, mbDst.UIDValidity, compactUIDSet(uids).String(), compactUIDSet(newUIDs).String())
4844// Move moves messages from the currently selected/active mailbox to a named mailbox.
4847func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) {
4854 name := p.xmailbox()
4857 name = xcheckmailboxname(name, true)
4860 xuserErrorf("mailbox open in read-only mode")
4864 var uids []store.UID
4866 var mbDst store.Mailbox
4867 var uidFirst store.UID
4868 var modseq store.ModSeq
4870 var cleanupIDs []int64
4872 for _, id := range cleanupIDs {
4873 p := c.account.MessagePath(id)
4875 c.xsanity(err, "removing destination message file %v", p)
4879 c.account.WithWLock(func() {
4880 var changes []store.Change
4882 c.xdbwrite(func(tx *bstore.Tx) {
4883 mbSrc := c.xmailboxID(tx, c.mailboxID) // Validate.
4884 mbDst = c.xmailbox(tx, name, "TRYCREATE")
4885 if mbDst.ID == c.mailboxID {
4886 xuserErrorf("cannot move to currently selected mailbox")
4889 uids = c.gatherCopyMoveUIDs(tx, isUID, nums)
4892 xuserErrorf("no matching messages to move")
4895 uidFirst = mbDst.UIDNext
4897 // Assign a new modseq, for the new records and for the expunged records.
4899 modseq, err = c.account.NextModSeq(tx)
4900 xcheckf(err, "assigning next modseq")
4902 // Make query selecting messages to move.
4903 q := bstore.QueryTx[store.Message](tx)
4904 q.FilterNonzero(store.Message{MailboxID: mbSrc.ID})
4905 q.FilterEqual("UID", slicesAny(uids)...)
4906 q.FilterEqual("Expunged", false)
4909 newIDs, chl := c.xmoveMessages(tx, q, len(uids), modseq, &mbSrc, &mbDst)
4910 changes = append(changes, chl...)
4916 c.broadcast(changes)
4921 newUIDs := numSet{ranges: []numRange{{setNumber{number: uint32(uidFirst)}, &setNumber{number: uint32(mbDst.UIDNext - 1)}}}}
4922 c.xbwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), newUIDs.String())
4923 qresync := c.enabled[capQresync]
4924 var vanishedUIDs numSet
4925 for i := range uids {
4929 vanishedUIDs.append(uint32(uids[i]))
4933 seq := c.xsequence(uids[i])
4934 c.sequenceRemove(seq, uids[i])
4936 vanishedUIDs.append(uint32(uids[i]))
4938 c.xbwritelinef("* %d EXPUNGE", seq)
4941 if !vanishedUIDs.empty() {
4943 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
4944 c.xbwritelinef("* VANISHED %s", s)
4950 c.xwriteresultf("%s OK [HIGHESTMODSEQ %d] move", tag, modseq.Client())
4956// q must yield messages from a single mailbox.
4957func (c *conn) xmoveMessages(tx *bstore.Tx, q *bstore.Query[store.Message], expectCount int, modseq store.ModSeq, mbSrc, mbDst *store.Mailbox) (newIDs []int64, changes []store.Change) {
4958 newIDs = make([]int64, 0, expectCount)
4964 for _, id := range newIDs {
4965 p := c.account.MessagePath(id)
4967 c.xsanity(err, "removing added message file %v", p)
4972 mbSrc.ModSeq = modseq
4973 mbDst.ModSeq = modseq
4978 err := jf.CloseDiscard()
4979 c.log.Check(err, "closing junk filter after error")
4983 accConf, _ := c.account.Conf()
4985 changeRemoveUIDs := store.ChangeRemoveUIDs{
4986 MailboxID: mbSrc.ID,
4989 changes = make([]store.Change, 0, expectCount+4) // mbsrc removeuids, mbsrc counts, mbdst counts, mbdst keywords
4991 nkeywords := len(mbDst.Keywords)
4995 xcheckf(err, "listing messages to move")
4997 if expectCount > 0 && len(l) != expectCount {
4998 xcheckf(fmt.Errorf("moved %d messages, expected %d", len(l), expectCount), "move messages")
5001 // For newly created message directories that we sync after hardlinking/copying files.
5002 syncDirs := map[string]struct{}{}
5004 for _, om := range l {
5006 nm.MailboxID = mbDst.ID
5007 nm.UID = mbDst.UIDNext
5008 err := mbDst.UIDNextAdd(1)
5009 xcheckf(err, "adding uid")
5011 nm.CreateSeq = modseq
5013 if nm.IsReject && nm.MailboxDestinedID != 0 {
5014 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
5015 // is used for reputation calculation during future deliveries.
5016 nm.MailboxOrigID = nm.MailboxDestinedID
5021 nm.JunkFlagsForMailbox(*mbDst, accConf)
5023 err = tx.Update(&nm)
5024 xcheckf(err, "updating message with new mailbox")
5026 mbDst.Add(nm.MailboxCounts())
5028 mbSrc.Sub(om.MailboxCounts())
5032 om.TrainedJunk = nil
5033 err = tx.Insert(&om)
5034 xcheckf(err, "inserting expunged message in old mailbox")
5036 dstPath := c.account.MessagePath(om.ID)
5037 dstDir := filepath.Dir(dstPath)
5038 if _, ok := syncDirs[dstDir]; !ok {
5039 os.MkdirAll(dstDir, 0770)
5040 syncDirs[dstDir] = struct{}{}
5043 err = moxio.LinkOrCopy(c.log, dstPath, c.account.MessagePath(nm.ID), nil, false)
5044 xcheckf(err, "duplicating message in old mailbox for current sessions")
5045 newIDs = append(newIDs, nm.ID)
5046 // We don't sync the directory. In case of a crash and files disappearing, the
5047 // eraser will simply not find the file at next startup.
5049 err = tx.Insert(&store.MessageErase{ID: om.ID, SkipUpdateDiskUsage: true})
5050 xcheckf(err, "insert message erase")
5052 mbDst.Keywords, _ = store.MergeKeywords(mbDst.Keywords, nm.Keywords)
5054 if accConf.JunkFilter != nil && nm.NeedsTraining() {
5055 // Lazily open junk filter.
5057 jf, _, err = c.account.OpenJunkFilter(context.TODO(), c.log)
5058 xcheckf(err, "open junk filter")
5060 err := c.account.RetrainMessage(context.TODO(), c.log, tx, jf, &nm)
5061 xcheckf(err, "retrain message after moving")
5064 changeRemoveUIDs.UIDs = append(changeRemoveUIDs.UIDs, om.UID)
5065 changeRemoveUIDs.MsgIDs = append(changeRemoveUIDs.MsgIDs, om.ID)
5066 changes = append(changes, nm.ChangeAddUID(*mbDst))
5068 xcheckf(err, "move messages")
5070 for dir := range syncDirs {
5071 err := moxio.SyncDir(c.log, dir)
5072 xcheckf(err, "sync directory")
5075 changeRemoveUIDs.UIDNext = mbDst.UIDNext
5076 changeRemoveUIDs.MessageCountIMAP = mbDst.MessageCountIMAP()
5077 changeRemoveUIDs.Unseen = uint32(mbDst.MailboxCounts.Unseen)
5078 changes = append(changes, changeRemoveUIDs, mbSrc.ChangeCounts())
5080 err = tx.Update(mbSrc)
5081 xcheckf(err, "updating counts for inbox")
5083 changes = append(changes, mbDst.ChangeCounts())
5084 if len(mbDst.Keywords) > nkeywords {
5085 changes = append(changes, mbDst.ChangeKeywords())
5088 err = tx.Update(mbDst)
5089 xcheckf(err, "updating uidnext and counts in destination mailbox")
5094 xcheckf(err, "saving junk filter")
5101// Store sets a full set of flags, or adds/removes specific flags.
5104func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) {
5111 var unchangedSince *int64
5114 p.xtake("UNCHANGEDSINCE")
5121 c.xensureCondstore(nil)
5123 var plus, minus bool
5126 } else if p.take("-") {
5130 silent := p.take(".SILENT")
5132 var flagstrs []string
5133 if p.hasPrefix("(") {
5134 flagstrs = p.xflagList()
5136 flagstrs = append(flagstrs, p.xflag())
5138 flagstrs = append(flagstrs, p.xflag())
5144 xuserErrorf("mailbox open in read-only mode")
5147 flags, keywords, err := store.ParseFlagsKeywords(flagstrs)
5149 xuserErrorf("parsing flags: %v", err)
5151 var mask store.Flags
5153 mask, flags = flags, store.FlagsAll
5155 mask, flags = flags, store.Flags{}
5157 mask = store.FlagsAll
5160 var mb, origmb store.Mailbox
5161 var updated []store.Message
5162 var changed []store.Message // ModSeq more recent than unchangedSince, will be in MODIFIED response code, and we will send untagged fetch responses so client is up to date.
5163 var modseq store.ModSeq // Assigned when needed.
5164 modified := map[int64]bool{}
5166 c.account.WithWLock(func() {
5167 var mbKwChanged bool
5168 var changes []store.Change
5170 c.xdbwrite(func(tx *bstore.Tx) {
5171 mb = c.xmailboxID(tx, c.mailboxID) // Validate.
5174 uids := c.xnumSetEval(tx, isUID, nums)
5180 // Ensure keywords are in mailbox.
5182 mb.Keywords, mbKwChanged = store.MergeKeywords(mb.Keywords, keywords)
5184 err := tx.Update(&mb)
5185 xcheckf(err, "updating mailbox with keywords")
5189 q := bstore.QueryTx[store.Message](tx)
5190 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
5191 q.FilterEqual("UID", slicesAny(uids)...)
5192 q.FilterEqual("Expunged", false)
5193 err := q.ForEach(func(m store.Message) error {
5194 // Client may specify a message multiple times, but we only process it once.
../rfc/7162:823
5199 mc := m.MailboxCounts()
5201 origFlags := m.Flags
5202 m.Flags = m.Flags.Set(mask, flags)
5203 oldKeywords := slices.Clone(m.Keywords)
5205 m.Keywords, _ = store.RemoveKeywords(m.Keywords, keywords)
5207 m.Keywords, _ = store.MergeKeywords(m.Keywords, keywords)
5209 m.Keywords = keywords
5212 keywordsChanged := func() bool {
5213 sort.Strings(oldKeywords)
5214 n := slices.Clone(m.Keywords)
5216 return !slices.Equal(oldKeywords, n)
5219 // If the message has a more recent modseq than the check requires, we won't modify
5220 // it and report in the final command response.
5223 // unchangedSince 0 always fails the check, we don't turn it into 1 like with our
5224 // internal modseqs. RFC implies that is not required for non-system flags, but we
5226 if unchangedSince != nil && m.ModSeq.Client() > *unchangedSince {
5227 changed = append(changed, m)
5232 // It requires that we keep track of the flags we think the client knows (but only
5233 // on this connection). We don't track that. It also isn't clear why this is
5234 // allowed because it is skipping the condstore conditional check, and the new
5235 // combination of flags could be unintended.
5238 if origFlags == m.Flags && !keywordsChanged() {
5239 // Note: since we didn't update the modseq, we are not adding m.ID to "modified",
5240 // it would skip the modseq check above. We still add m to list of updated, so we
5241 // send an untagged fetch response. But we don't broadcast it.
5242 updated = append(updated, m)
5247 mb.Add(m.MailboxCounts())
5249 // Assign new modseq for first actual change.
5252 modseq, err = c.account.NextModSeq(tx)
5253 xcheckf(err, "next modseq")
5257 modified[m.ID] = true
5258 updated = append(updated, m)
5260 changes = append(changes, m.ChangeFlags(origFlags, mb))
5262 return tx.Update(&m)
5264 xcheckf(err, "storing flags in messages")
5266 if mb.MailboxCounts != origmb.MailboxCounts || modseq != 0 {
5267 err := tx.Update(&mb)
5268 xcheckf(err, "updating mailbox counts")
5270 if mb.MailboxCounts != origmb.MailboxCounts {
5271 changes = append(changes, mb.ChangeCounts())
5274 changes = append(changes, mb.ChangeKeywords())
5277 err = c.account.RetrainMessages(context.TODO(), c.log, tx, updated)
5278 xcheckf(err, "training messages")
5281 c.broadcast(changes)
5284 // In the RFC, the section about STORE/UID STORE says we must return MODSEQ when
5285 // UNCHANGEDSINCE was specified. It does not specify it in case UNCHANGEDSINCE
5286 // isn't specified. For that case it does say MODSEQ is needed in unsolicited
5287 // untagged fetch responses. Implying that solicited untagged fetch responses
5288 // should not include MODSEQ (why else mention unsolicited explicitly?). But, in
5289 // the introduction to CONDSTORE it does explicitly specify MODSEQ should be
5290 // included in untagged fetch responses at all times with CONDSTORE-enabled
5291 // connections. It would have been better if the command behaviour was specified in
5292 // the command section, not the introduction to the extension.
5295 if !silent || c.enabled[capCondstore] {
5296 for _, m := range updated {
5299 args = append(args, fmt.Sprintf("FLAGS %s", flaglist(m.Flags, m.Keywords).pack(c)))
5301 if c.enabled[capCondstore] {
5302 args = append(args, fmt.Sprintf("MODSEQ (%d)", m.ModSeq.Client()))
5307 // Ensure list is non-empty.
5309 args = append(args, fmt.Sprintf("UID %d", m.UID))
5311 c.xbwritelinef("* %d UIDFETCH (%s)", m.UID, strings.Join(args, " "))
5313 args = append([]string{fmt.Sprintf("UID %d", m.UID)}, args...)
5314 c.xbwritelinef("* %d FETCH (%s)", c.xsequence(m.UID), strings.Join(args, " "))
5319 // We don't explicitly send flags for failed updated with silent set. The regular
5320 // notification will get the flags to the client.
5323 if len(changed) == 0 {
5328 // Write unsolicited untagged fetch responses for messages that didn't pass the
5331 var mnums []store.UID
5332 for _, m := range changed {
5335 c.xbwritelinef("* %d UIDFETCH (FLAGS %s MODSEQ (%d))", m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
5337 c.xbwritelinef("* %d FETCH (UID %d FLAGS %s MODSEQ (%d))", c.xsequence(m.UID), m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
5340 mnums = append(mnums, m.UID)
5342 mnums = append(mnums, store.UID(c.xsequence(m.UID)))
5347 set := compactUIDSet(mnums)
5349 c.xwriteresultf("%s OK [MODIFIED %s] conditional store did not modify all", tag, set.String())