1package smtpserver
2
3import (
4 "context"
5 "fmt"
6 "log/slog"
7 "net"
8 "os"
9 "strings"
10 "time"
11
12 "github.com/mjl-/bstore"
13
14 "github.com/mjl-/mox/config"
15 "github.com/mjl-/mox/dkim"
16 "github.com/mjl-/mox/dmarc"
17 "github.com/mjl-/mox/dmarcrpt"
18 "github.com/mjl-/mox/dns"
19 "github.com/mjl-/mox/dnsbl"
20 "github.com/mjl-/mox/iprev"
21 "github.com/mjl-/mox/message"
22 "github.com/mjl-/mox/mlog"
23 "github.com/mjl-/mox/mox-"
24 "github.com/mjl-/mox/publicsuffix"
25 "github.com/mjl-/mox/smtp"
26 "github.com/mjl-/mox/store"
27 "github.com/mjl-/mox/subjectpass"
28 "github.com/mjl-/mox/tlsrpt"
29)
30
31type delivery struct {
32 tls bool
33 m *store.Message
34 dataFile *os.File
35 smtpRcptTo smtp.Path // As used in SMTP, possibly address of alias.
36 deliverTo smtp.Path // To deliver to, either smtpRcptTo or an alias member address.
37 destination config.Destination
38 canonicalAddress string
39 acc *store.Account
40 msgTo []message.Address
41 msgCc []message.Address
42 msgFrom smtp.Address
43 dnsBLs []dns.Domain
44 dmarcUse bool
45 dmarcResult dmarc.Result
46 dkimResults []dkim.Result
47 iprevStatus iprev.Status
48 smtputf8 bool
49}
50
51type analysis struct {
52 d delivery
53 accept bool
54 mailbox string // Where to deliver to.
55 mailboxDestined string // Non-empty when message would normally be delivered to mailbox, but introbox or rejects rule affected delivery.
56 code int
57 secode string
58 userError bool
59 errmsg string
60 err error // For our own logging, not sent to remote.
61 dmarcReport *dmarcrpt.Feedback // Validated DMARC aggregate report, not yet stored.
62 tlsReport *tlsrpt.Report // Validated TLS report, not yet stored.
63 reason string // If non-empty, reason for this decision. Values from reputationMethod and reason* below.
64 reasonText []string // Additional details for reason, human-readable, added to X-Mox-Reason header.
65 dmarcOverrideReason string // If set, one of dmarcrpt.PolicyOverride
66 // Additional headers to add during delivery. Used for reasons a message to a
67 // dmarc/tls reporting address isn't processed.
68 headers string
69}
70
71const (
72 reasonListAllow = "list-allow"
73 reasonDMARCPolicy = "dmarc-policy"
74 reasonReputationError = "reputation-error"
75 reasonReporting = "reporting"
76 reasonSPFPolicy = "spf-policy"
77 reasonJunkClassifyError = "junk-classify-error"
78 reasonJunkFilterError = "junk-filter-error"
79 reasonGiveSubjectpass = "give-subjectpass"
80 reasonNoBadSignals = "no-bad-signals"
81 reasonJunkContent = "junk-content"
82 reasonJunkContentStrict = "junk-content-strict"
83 reasonDNSBlocklisted = "dns-blocklisted"
84 reasonSubjectpass = "subjectpass"
85 reasonSubjectpassError = "subjectpass-error"
86 reasonIPrev = "iprev" // No or mild junk reputation signals, and bad iprev.
87 reasonHighRate = "high-rate" // Too many messages, not added to rejects.
88 reasonMsgAuthRequired = "msg-auth-required"
89)
90
91func isListDomain(d delivery, ld dns.Domain) bool {
92 if d.m.MailFromValidated && ld.Name() == d.m.MailFromDomain {
93 return true
94 }
95 for _, r := range d.dkimResults {
96 if r.Status == dkim.StatusPass && r.Sig.Domain == ld {
97 return true
98 }
99 }
100 return false
101}
102
103func analyze(ctx context.Context, log mlog.Log, resolver dns.Resolver, d delivery) analysis {
104 var headers string
105
106 var reasonText []string
107 addReasonText := func(format string, args ...any) {
108 s := fmt.Sprintf(format, args...)
109 reasonText = append(reasonText, s)
110 }
111
112 // We don't want to let a single IP or network deliver too many messages to an
113 // account. They may fill up the mailbox, either with messages that have to be
114 // purged, or by filling the disk. We check both cases for IP's and networks.
115 var rateError bool // Whether returned error represents a rate error.
116 err := d.acc.DB.Read(ctx, func(tx *bstore.Tx) (retErr error) {
117 now := time.Now()
118 defer func() {
119 log.Debugx("checking message and size delivery rates", retErr, slog.Duration("duration", time.Since(now)))
120 }()
121
122 checkCount := func(msg store.Message, window time.Duration, limit int) {
123 if retErr != nil {
124 return
125 }
126 q := bstore.QueryTx[store.Message](tx)
127 q.FilterNonzero(msg)
128 q.FilterGreater("Received", now.Add(-window))
129 q.FilterEqual("Expunged", false)
130 n, err := q.Count()
131 if err != nil {
132 retErr = err
133 return
134 }
135 if n >= limit {
136 rateError = true
137 retErr = fmt.Errorf("more than %d messages in past %s from your ip/network", limit, window)
138 }
139 }
140
141 checkSize := func(msg store.Message, window time.Duration, limit int64) {
142 if retErr != nil {
143 return
144 }
145 q := bstore.QueryTx[store.Message](tx)
146 q.FilterNonzero(msg)
147 q.FilterGreater("Received", now.Add(-window))
148 q.FilterEqual("Expunged", false)
149 size := d.m.Size
150 err := q.ForEach(func(v store.Message) error {
151 size += v.Size
152 return nil
153 })
154 if err != nil {
155 retErr = err
156 return
157 }
158 if size > limit {
159 rateError = true
160 retErr = fmt.Errorf("more than %d bytes in past %s from your ip/network", limit, window)
161 }
162 }
163
164 // todo future: make these configurable
165 // todo: should we have a limit for forwarded messages? they are stored with empty RemoteIPMasked*
166
167 const day = 24 * time.Hour
168 checkCount(store.Message{RemoteIPMasked1: d.m.RemoteIPMasked1}, time.Minute, limitIPMasked1MessagesPerMinute)
169 checkCount(store.Message{RemoteIPMasked1: d.m.RemoteIPMasked1}, day, 20*500)
170 checkCount(store.Message{RemoteIPMasked2: d.m.RemoteIPMasked2}, time.Minute, 1500)
171 checkCount(store.Message{RemoteIPMasked2: d.m.RemoteIPMasked2}, day, 20*1500)
172 checkCount(store.Message{RemoteIPMasked3: d.m.RemoteIPMasked3}, time.Minute, 4500)
173 checkCount(store.Message{RemoteIPMasked3: d.m.RemoteIPMasked3}, day, 20*4500)
174
175 const MB = 1024 * 1024
176 checkSize(store.Message{RemoteIPMasked1: d.m.RemoteIPMasked1}, time.Minute, limitIPMasked1SizePerMinute)
177 checkSize(store.Message{RemoteIPMasked1: d.m.RemoteIPMasked1}, day, 3*1000*MB)
178 checkSize(store.Message{RemoteIPMasked2: d.m.RemoteIPMasked2}, time.Minute, 3000*MB)
179 checkSize(store.Message{RemoteIPMasked2: d.m.RemoteIPMasked2}, day, 3*3000*MB)
180 checkSize(store.Message{RemoteIPMasked3: d.m.RemoteIPMasked3}, time.Minute, 9000*MB)
181 checkSize(store.Message{RemoteIPMasked3: d.m.RemoteIPMasked3}, day, 3*9000*MB)
182
183 return retErr
184 })
185 if err != nil && !rateError {
186 log.Errorx("checking delivery rates", err)
187 metricDelivery.WithLabelValues("checkrates", "").Inc()
188 addReasonText("checking delivery rates: %v", err)
189 return analysis{d, false, "", "", smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError, reasonText, "", headers}
190 } else if err != nil {
191 log.Debugx("refusing due to high delivery rate", err)
192 metricDelivery.WithLabelValues("highrate", "").Inc()
193 addReasonText("high delivery rate")
194 return analysis{d, false, "", "", smtp.C452StorageFull, smtp.SeMailbox2Full2, true, err.Error(), err, nil, nil, reasonHighRate, reasonText, "", headers}
195 }
196
197 var mailboxDestined string // Only set when we change mailbox, e.g. due to introbox or reject.
198 mailbox := d.destination.Mailbox
199 if mailbox == "" {
200 mailbox = "Inbox"
201 }
202
203 // If destination mailbox has a mailing list domain (for SPF/DKIM) configured,
204 // check it for a pass.
205 rs := store.MessageRuleset(log, d.destination, d.m, d.m.MsgPrefix, d.dataFile)
206 if rs != nil {
207 mailbox = rs.Mailbox
208 }
209 if rs != nil && !rs.ListAllowDNSDomain.IsZero() {
210 // todo: on temporary failures, reject temporarily?
211 if isListDomain(d, rs.ListAllowDNSDomain) {
212 addReasonText("validated message from a configured mailing list")
213 d.m.IsMailingList = true
214 return analysis{
215 d: d,
216 accept: true,
217 mailbox: mailbox,
218 reason: reasonListAllow,
219 reasonText: reasonText,
220 dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList),
221 headers: headers,
222 }
223 }
224 }
225
226 var dmarcOverrideReason string
227
228 // For forwarded messages, we have different junk analysis. We don't reject for
229 // failing DMARC, and we clear fields that could implicate the forwarding mail
230 // server during future classifications on incoming messages (the forwarding mail
231 // server isn't responsible for the message).
232 if rs != nil && rs.IsForward {
233 d.dmarcUse = false
234 d.m.IsForward = true
235 d.m.RemoteIPMasked1 = ""
236 d.m.RemoteIPMasked2 = ""
237 d.m.RemoteIPMasked3 = ""
238 d.m.OrigEHLODomain = d.m.EHLODomain
239 d.m.EHLODomain = ""
240 d.m.MailFromDomain = "" // Still available in MailFrom.
241 d.m.OrigDKIMDomains = d.m.DKIMDomains
242 dkimdoms := []string{}
243 for _, dom := range d.m.DKIMDomains {
244 if dom != rs.VerifiedDNSDomain.Name() {
245 dkimdoms = append(dkimdoms, dom)
246 }
247 }
248 d.m.DKIMDomains = dkimdoms
249 dmarcOverrideReason = string(dmarcrpt.PolicyOverrideForwarded)
250 log.Info("forwarded message, clearing identifying signals of forwarding mail server")
251 addReasonText("ruleset indicates forwarded message")
252 }
253
254 reject := func(code int, secode string, errmsg string, err error, reason string) analysis {
255 accept := false
256 // mailboxDestined may already have been set because of Introbox.
257 if mailboxDestined == "" {
258 mailboxDestined = mailbox
259 }
260 if rs != nil && rs.AcceptRejectsToMailbox != "" {
261 accept = true
262 mailbox = rs.AcceptRejectsToMailbox
263 d.m.IsReject = true
264 // Don't draw attention, but don't go so far as to mark as junk.
265 d.m.Seen = true
266 log.Info("accepting reject to configured mailbox due to ruleset")
267 addReasonText("accepting reject to mailbox due to ruleset")
268 } else {
269 conf, _ := d.acc.Conf()
270 mailbox = conf.RejectsMailbox
271 }
272 return analysis{d, accept, mailbox, mailboxDestined, code, secode, err == nil, errmsg, err, nil, nil, reason, reasonText, dmarcOverrideReason, headers}
273 }
274
275 if d.dmarcUse && d.dmarcResult.Reject {
276 addReasonText("message does not pass domain dmarc policy which asks to reject")
277 return reject(smtp.C550MailboxUnavail, smtp.SePol7MultiAuthFails26, "rejecting per dmarc policy", nil, reasonDMARCPolicy)
278 } else if !d.dmarcUse {
279 addReasonText("not using any dmarc result")
280 } else {
281 addReasonText("dmarc ok")
282 }
283 // todo: should we also reject messages that have a dmarc pass but an spf record "v=spf1 -all"? suggested by m3aawg best practices.
284
285 // If destination is the DMARC reporting mailbox, do additional checks and keep
286 // track of the report. We'll check reputation, defaulting to accept.
287 var dmarcReport *dmarcrpt.Feedback
288 if d.destination.DMARCReports {
289 // Messages with DMARC aggregate reports must have a DMARC pass. ../rfc/7489:1866
290 if d.dmarcResult.Status != dmarc.StatusPass {
291 log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report")
292 headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n"
293 } else if report, err := dmarcrpt.ParseMessageReport(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
294 log.Infox("parsing dmarc aggregate report", err)
295 headers += "X-Mox-DMARCReport-Error: could not parse report\r\n"
296 } else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {
297 log.Infox("parsing domain in dmarc aggregate report", err)
298 headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n"
299 } else if _, ok := mox.Conf.Domain(d); !ok {
300 log.Info("dmarc aggregate report for domain not configured, ignoring", slog.Any("domain", d))
301 headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n"
302 } else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {
303 log.Info("dmarc aggregate report with end date in the future, ignoring", slog.Any("domain", d), slog.Time("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))
304 headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n"
305 } else {
306 dmarcReport = report
307 }
308 }
309
310 // Similar to DMARC reporting, we check for the required DKIM. We'll check
311 // reputation, defaulting to accept.
312 var tlsReport *tlsrpt.Report
313 if d.destination.HostTLSReports || d.destination.DomainTLSReports {
314 matchesDomain := func(sigDomain dns.Domain) bool {
315 // RFC seems to require exact DKIM domain match with submitt and message From, we
316 // also allow msgFrom to be subdomain. ../rfc/8460:322
317 return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, log.Logger, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, log.Logger, sigDomain)
318 }
319 // Valid DKIM signature for domain must be present. We take "valid" to assume
320 // "passing", not "syntactically valid". We also check for "tlsrpt" as service.
321 // This check is optional, but if anyone goes through the trouble to explicitly
322 // list allowed services, they would be surprised to see them ignored.
323 // ../rfc/8460:320
324 ok := false
325 for _, r := range d.dkimResults {
326 // The record should have an allowed service "tlsrpt". The RFC mentions it as if
327 // the service must be specified explicitly, but the default allowed services for a
328 // DKIM record are "*", which includes "tlsrpt". Unless a DKIM record explicitly
329 // specifies services (e.g. s=email), a record will work for TLS reports. The DKIM
330 // records seen used for TLS reporting in the wild don't explicitly set "s" for
331 // services.
332 // ../rfc/8460:326
333 if r.Status == dkim.StatusPass && matchesDomain(r.Sig.Domain) && r.Sig.Length < 0 && r.Record.ServiceAllowed("tlsrpt") {
334 ok = true
335 break
336 }
337 }
338
339 if !ok {
340 log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report")
341 headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n"
342 } else if reportJSON, err := tlsrpt.ParseMessage(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
343 log.Infox("parsing tls report", err)
344 headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n"
345 } else {
346 var known bool
347 for _, p := range reportJSON.Policies {
348 log.Info("tlsrpt policy domain", slog.String("domain", p.Policy.Domain))
349 if d, err := dns.ParseDomain(p.Policy.Domain); err != nil {
350 log.Infox("parsing domain in tls report", err)
351 } else if _, ok := mox.Conf.Domain(d); ok || d == mox.Conf.Static.HostnameDomain {
352 known = true
353 break
354 }
355 }
356 if !known {
357 log.Info("tls report without one of configured domains, ignoring")
358 headers += "X-Mox-TLSReport-Error: report for unknown domain\r\n"
359 } else {
360 report := reportJSON.Convert()
361 tlsReport = &report
362 }
363 }
364 }
365
366 // We may have to reject messages that don't pass a relaxed aligned SPF and/or DKIM
367 // check. Useful for services with autoresponders.
368 if d.destination.MessageAuthRequiredSMTPError != "" && !d.m.MsgFromValidated {
369 code := smtp.C550MailboxUnavail
370 msg := d.destination.MessageAuthRequiredSMTPError
371 if d.dmarcResult.Status == dmarc.StatusTemperror {
372 code = smtp.C451LocalErr
373 msg = "transient verification error: " + msg
374 }
375 addReasonText("message does not pass required aligned spf and/or dkim check required for destination")
376 return reject(code, smtp.SePol7MultiAuthFails26, msg, nil, reasonMsgAuthRequired)
377 }
378
379 // Determine if message is acceptable based on DMARC domain, DKIM identities, or
380 // host-based reputation.
381 var isjunk *bool
382 var conclusive bool
383 var method reputationMethod
384 var reason string
385 d.acc.WithRLock(func() {
386 err = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {
387 var mailboxID int64 = -1
388 mb, err := d.acc.MailboxFind(tx, mailbox)
389 if err != nil {
390 return fmt.Errorf("finding destination mailbox: %w", err)
391 }
392 if mb != nil {
393 mailboxID = mb.ID
394 }
395
396 var text string
397 isjunk, conclusive, method, text, err = reputation(tx, log, d.m, mailboxID, d.smtputf8)
398 reason = string(method)
399 s := "address/dkim/spf/ip-based reputation ("
400 if isjunk != nil && *isjunk {
401 s += "junk, "
402 } else if isjunk != nil && !*isjunk {
403 s += "nonjunk, "
404 }
405 if conclusive {
406 s += "conclusive"
407 } else {
408 s += "inconclusive"
409 }
410 s += ", " + text + ")"
411 addReasonText("%s", s)
412 return err
413 })
414 })
415 if err != nil {
416 log.Infox("determining reputation", err, slog.Any("message", d.m))
417 addReasonText("determining reputation: %v", err)
418 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError)
419 }
420 log.Info("reputation analyzed",
421 slog.Bool("conclusive", conclusive),
422 slog.Any("isjunk", isjunk),
423 slog.String("method", string(method)))
424
425 // todo: we may want to add an Introbox field to rulesets, to enable it for destination mailboxes other than Inbox
426 // todo: we may want to look at referenced message-id's (from the thread), see if this message-from address was in a to/cc header in a message for the same mailbox the account marked as non-junk, and allow the message through. should help for regular messages (not intended for introbox) too.
427 conf, _ := d.acc.Conf()
428 introbox := mailbox == "Inbox" && conf.Introbox != "" && method != methodMsgfromFull && method != methodMsgtoFull && !d.m.IsForward && dmarcReport == nil && tlsReport == nil
429 if introbox {
430 mailbox, mailboxDestined = conf.Introbox, mailbox
431 log.Info("delivering message without established reputation to introbox", slog.String("mailbox", mailbox), slog.String("method", string(method)))
432 }
433
434 if conclusive {
435 if !*isjunk {
436 return analysis{
437 d: d,
438 accept: true,
439 mailbox: mailbox,
440 mailboxDestined: mailboxDestined,
441 dmarcReport: dmarcReport,
442 tlsReport: tlsReport,
443 reason: reason,
444 reasonText: reasonText,
445 dmarcOverrideReason: dmarcOverrideReason,
446 headers: headers,
447 }
448 }
449 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, string(method))
450 } else if dmarcReport != nil || tlsReport != nil {
451 log.Info("accepting message with dmarc aggregate report or tls report without reputation")
452 addReasonText("message inconclusive reputation but with dmarc or tls report")
453 return analysis{
454 d: d,
455 accept: true,
456 mailbox: mailbox,
457 mailboxDestined: mailboxDestined,
458 dmarcReport: dmarcReport,
459 tlsReport: tlsReport,
460 reason: reasonReporting,
461 reasonText: reasonText,
462 dmarcOverrideReason: dmarcOverrideReason,
463 headers: headers,
464 }
465 }
466 // If there was no previous message from sender or its domain, and we have an SPF
467 // (soft)fail, reject the message.
468 switch method {
469 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:
470 switch d.m.MailFromValidation {
471 case store.ValidationFail, store.ValidationSoftfail:
472 addReasonText("no previous message from sender domain and spf result is (soft)fail")
473 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonSPFPolicy)
474 }
475 }
476
477 // Senders without reputation and without iprev pass, are likely spam.
478 var suspiciousIPrevFail bool
479 switch method {
480 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:
481 suspiciousIPrevFail = d.iprevStatus != iprev.StatusPass
482 }
483 if suspiciousIPrevFail {
484 addReasonText("suspicious iprev failure")
485 }
486
487 // With already a mild junk signal, an iprev fail on top is enough to reject.
488 if suspiciousIPrevFail && isjunk != nil && *isjunk {
489 addReasonText("message has a mild junk signal and mismatching reverse ip")
490 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonIPrev)
491 }
492
493 var subjectpassKey string
494 if conf.SubjectPass.Period > 0 {
495 subjectpassKey, err = d.acc.Subjectpass(d.canonicalAddress)
496 if err != nil {
497 log.Errorx("get key for verifying subject token", err)
498 addReasonText("subject pass error: %v", err)
499 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError)
500 }
501 err = subjectpass.Verify(log.Logger, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period)
502 pass := err == nil
503 log.Infox("pass by subject token", err, slog.Bool("pass", pass))
504 if pass {
505 addReasonText("message has valid subjectpass token in subject")
506 return analysis{
507 d: d,
508 accept: true,
509 mailbox: mailbox,
510 mailboxDestined: mailboxDestined,
511 reason: reasonSubjectpass,
512 reasonText: reasonText,
513 dmarcOverrideReason: dmarcOverrideReason,
514 headers: headers,
515 }
516 }
517 }
518
519 reason = reasonNoBadSignals
520 accept := true
521 var junkSubjectpass bool
522 f, jf, err := d.acc.OpenJunkFilter(ctx, log)
523 if err == nil {
524 defer func() {
525 err := f.Close()
526 log.Check(err, "closing junkfilter")
527 }()
528 result, err := f.ClassifyMessageReader(ctx, store.FileMsgReader(d.m.MsgPrefix, d.dataFile), d.m.Size)
529 if err != nil {
530 log.Errorx("testing for spam", err)
531 addReasonText("classify message error: %v", err)
532 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkClassifyError)
533 }
534 // todo: if isjunk is not nil (i.e. there was inconclusive reputation), use it in the probability calculation. give reputation a score of 0.25 or .75 perhaps?
535 // todo: if there aren't enough historic messages, we should just let messages in.
536 // todo: we could require nham and nspam to be above a certain number when there were plenty of words in the message, and in the database. can indicate a spammer is misspelling words. however, it can also mean a message in a different language/script...
537
538 // If we don't accept, we may still respond with a "subjectpass" hint below.
539 // We add some jitter to the threshold we use. So we don't act as too easy an
540 // oracle for words that are a strong indicator of haminess.
541 // todo: we should rate-limit uses of the junkfilter.
542 jitter := (jitterRand.Float64() - 0.5) / 10
543 threshold := jf.Threshold + jitter
544
545 rcptToMatch := func(l []message.Address) bool {
546 // todo: we use Go's net/mail to parse message header addresses. it does not allow empty quoted strings (contrary to spec), leaving To empty. so we don't verify To address for that unusual case for now. ../rfc/5322:961 ../rfc/5322:743
547 if d.smtpRcptTo.Localpart == "" {
548 return true
549 }
550 for _, a := range l {
551 dom, err := dns.ParseDomain(a.Host)
552 if err != nil {
553 continue
554 }
555 lp, err := smtp.ParseLocalpart(a.User)
556 if err == nil && dom == d.smtpRcptTo.IPDomain.Domain && lp == d.smtpRcptTo.Localpart {
557 return true
558 }
559 }
560 return false
561 }
562
563 // todo: some of these checks should also apply for reputation-based analysis with a weak signal, e.g. verified dkim/spf signal from new domain.
564 // With an iprev fail, non-TLS connection or our address not in To/Cc header, we set a higher bar for content.
565 reason = reasonJunkContent
566 var thresholdRemark string
567 if suspiciousIPrevFail && threshold > 0.25 {
568 threshold = 0.25
569 log.Info("setting junk threshold due to iprev fail", slog.Float64("threshold", threshold))
570 reason = reasonJunkContentStrict
571 thresholdRemark = " (stricter due to reverse ip mismatch)"
572 } else if !d.tls && threshold > 0.25 {
573 threshold = 0.25
574 log.Info("setting junk threshold due to plaintext smtp", slog.Float64("threshold", threshold))
575 reason = reasonJunkContentStrict
576 thresholdRemark = " (stricter due to missing tls)"
577 } else if (rs == nil || !rs.IsForward) && threshold > 0.25 && !rcptToMatch(d.msgTo) && !rcptToMatch(d.msgCc) {
578 // A common theme in junk messages is your recipient address not being in the To/Cc
579 // headers. We may be in Bcc, but that's unusual for first-time senders. Some
580 // providers (e.g. gmail) does not DKIM-sign Bcc headers, so junk messages can be
581 // sent with matching Bcc headers. We don't get here for known senders.
582 threshold = 0.25
583 log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", slog.Float64("threshold", threshold))
584 reason = reasonJunkContentStrict
585 thresholdRemark = " (stricter due to recipient address not in to/cc header)"
586 }
587 accept = result.Probability <= threshold || (!result.Significant && !suspiciousIPrevFail)
588 junkSubjectpass = result.Probability < threshold-0.2
589 log.Info("content analyzed",
590 slog.Bool("accept", accept),
591 slog.Float64("contentprob", result.Probability),
592 slog.Bool("contentsignificant", result.Significant),
593 slog.Bool("subjectpass", junkSubjectpass))
594
595 var s strings.Builder
596 s.WriteString("content: ")
597 if accept {
598 s.WriteString("not junk")
599 } else {
600 s.WriteString("junk")
601 }
602 if !result.Significant {
603 s.WriteString(" (not significant)")
604 }
605 s.WriteString(fmt.Sprintf(", spamscore %.2f, threshold %.2f%s", result.Probability, threshold, thresholdRemark))
606 s.WriteString(" (ham words: ")
607 for i, w := range result.Hams {
608 if i > 0 {
609 s.WriteString(", ")
610 }
611 word := w.Word
612 if !d.smtputf8 && !isASCII(word) {
613 word = "(non-ascii)"
614 }
615 s.WriteString(fmt.Sprintf("%s %.3f", word, w.Score))
616 }
617 s.WriteString("), (spam words: ")
618 for i, w := range result.Spams {
619 if i > 0 {
620 s.WriteString(", ")
621 }
622 word := w.Word
623 if !d.smtputf8 && !isASCII(word) {
624 word = "(non-ascii)"
625 }
626 s.WriteString(fmt.Sprintf("%s %.3f", word, w.Score))
627 }
628 s.WriteString(")")
629 addReasonText("%s", s.String())
630 } else if err != store.ErrNoJunkFilter {
631 log.Errorx("open junkfilter", err)
632 addReasonText("open junkfilter: %v", err)
633 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError)
634 } else {
635 addReasonText("no junk filter configured")
636 }
637
638 // If content looks good, we'll still look at DNS block lists for a reason to
639 // reject. We normally won't get here if we've communicated with this sender
640 // before.
641 var dnsblocklisted bool
642 if accept {
643 blocked := func(zone dns.Domain) bool {
644 dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second)
645 defer dnsblcancel()
646 if !checkDNSBLHealth(dnsblctx, log, resolver, zone) {
647 log.Info("dnsbl not healthy, skipping", slog.Any("zone", zone))
648 return false
649 }
650
651 status, expl, err := dnsbl.Lookup(dnsblctx, log.Logger, resolver, zone, net.ParseIP(d.m.RemoteIP))
652 dnsblcancel()
653 if status == dnsbl.StatusFail {
654 log.Info("rejecting due to listing in dnsbl", slog.Any("zone", zone), slog.String("explanation", expl))
655 return true
656 } else if err != nil {
657 log.Infox("dnsbl lookup", err, slog.Any("zone", zone), slog.Any("status", status))
658 }
659 return false
660 }
661
662 // Note: We don't check in parallel, we are in no hurry to accept possible spam.
663 for _, zone := range d.dnsBLs {
664 if blocked(zone) {
665 accept = false
666 dnsblocklisted = true
667 reason = reasonDNSBlocklisted
668 addReasonText("dnsbl: ip %s listed in dnsbl %s", d.m.RemoteIP, zone.XName(d.smtputf8))
669 break
670 }
671 }
672 if !dnsblocklisted && len(d.dnsBLs) > 0 {
673 addReasonText("remote ip not blocklisted")
674 }
675 }
676
677 if accept {
678 addReasonText("no known reputation and no bad signals")
679 return analysis{
680 d: d,
681 accept: true,
682 mailbox: mailbox,
683 mailboxDestined: mailboxDestined,
684 reason: reasonNoBadSignals,
685 reasonText: reasonText,
686 dmarcOverrideReason: dmarcOverrideReason,
687 headers: headers,
688 }
689 }
690
691 if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {
692 log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation")
693 pass := subjectpass.Generate(log.Logger, d.msgFrom, []byte(subjectpassKey), time.Now())
694 addReasonText("reject with request to try again with subjectpass token in subject")
695 return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass)
696 }
697
698 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reason)
699}
700
701func isASCII(s string) bool {
702 for _, b := range []byte(s) {
703 if b >= 0x80 {
704 return false
705 }
706 }
707 return true
708}
709