1package smtpserver

2

3import (

4 "context"

5 "fmt"

6 "net"

7 "os"

8 "strings"

9 "time"

10

11 "github.com/mjl-/bstore"

12

13 "github.com/mjl-/mox/dkim"

14 "github.com/mjl-/mox/dmarc"

15 "github.com/mjl-/mox/dmarcrpt"

16 "github.com/mjl-/mox/dns"

17 "github.com/mjl-/mox/dnsbl"

18 "github.com/mjl-/mox/iprev"

19 "github.com/mjl-/mox/mlog"

20 "github.com/mjl-/mox/mox-"

21 "github.com/mjl-/mox/publicsuffix"

22 "github.com/mjl-/mox/smtp"

23 "github.com/mjl-/mox/store"

24 "github.com/mjl-/mox/subjectpass"

25 "github.com/mjl-/mox/tlsrpt"

26)

27

28type delivery struct {

29 m *store.Message

30 dataFile *os.File

31 rcptAcc rcptAccount

32 acc *store.Account

33 msgFrom smtp.Address

34 dnsBLs []dns.Domain

35 dmarcUse bool

36 dmarcResult dmarc.Result

37 dkimResults []dkim.Result

38 iprevStatus iprev.Status

39}

40

41type analysis struct {

42 accept bool

43 mailbox string

44 code int

45 secode string

46 userError bool

47 errmsg string

48 err error // For our own logging, not sent to remote.

49 dmarcReport *dmarcrpt.Feedback // Validated DMARC aggregate report, not yet stored.

50 tlsReport *tlsrpt.Report // Validated TLS report, not yet stored.

51 reason string // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens.

52 dmarcOverrideReason string // If set, one of dmarcrpt.PolicyOverride

53 // Additional headers to add during delivery. Used for reasons a message to a

54 // dmarc/tls reporting address isn't processed.

55 headers string

56}

57

58const (

59 reasonListAllow = "list-allow"

60 reasonDMARCPolicy = "dmarc-policy"

61 reasonReputationError = "reputation-error"

62 reasonReporting = "reporting"

63 reasonSPFPolicy = "spf-policy"

64 reasonJunkClassifyError = "junk-classify-error"

65 reasonJunkFilterError = "junk-filter-error"

66 reasonGiveSubjectpass = "give-subjectpass"

67 reasonNoBadSignals = "no-bad-signals"

68 reasonJunkContent = "junk-content"

69 reasonJunkContentStrict = "junk-content-strict"

70 reasonDNSBlocklisted = "dns-blocklisted"

71 reasonSubjectpass = "subjectpass"

72 reasonSubjectpassError = "subjectpass-error"

73 reasonIPrev = "iprev" // No or mild junk reputation signals, and bad iprev.

74)

75

76func isListDomain(d delivery, ld dns.Domain) bool {

77 if d.m.MailFromValidated && ld.Name() == d.m.MailFromDomain {

78 return true

79 }

80 for _, r := range d.dkimResults {

81 if r.Status == dkim.StatusPass && r.Sig.Domain == ld {

82 return true

83 }

84 }

85 return false

86}

87

88func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delivery) analysis {

89 var headers string

90

91 mailbox := d.rcptAcc.destination.Mailbox

92 if mailbox == "" {

93 mailbox = "Inbox"

94 }

95

96 // If destination mailbox has a mailing list domain (for SPF/DKIM) configured,

97 // check it for a pass.

98 rs := store.MessageRuleset(log, d.rcptAcc.destination, d.m, d.m.MsgPrefix, d.dataFile)

99 if rs != nil {

100 mailbox = rs.Mailbox

101 }

102 if rs != nil && !rs.ListAllowDNSDomain.IsZero() {

103 // todo: on temporary failures, reject temporarily?

104 if isListDomain(d, rs.ListAllowDNSDomain) {

105 d.m.IsMailingList = true

106 return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow, dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList), headers: headers}

107 }

108 }

109

110 var dmarcOverrideReason string

111

112 // For forwarded messages, we have different junk analysis. We don't reject for

113 // failing DMARC, and we clear fields that could implicate the forwarding mail

114 // server during future classifications on incoming messages (the forwarding mail

115 // server isn't responsible for the message).

116 if rs != nil && rs.IsForward {

117 d.dmarcUse = false

118 d.m.IsForward = true

119 d.m.RemoteIPMasked1 = ""

120 d.m.RemoteIPMasked2 = ""

121 d.m.RemoteIPMasked3 = ""

122 d.m.OrigEHLODomain = d.m.EHLODomain

123 d.m.EHLODomain = ""

124 d.m.MailFromDomain = "" // Still available in MailFrom.

125 d.m.OrigDKIMDomains = d.m.DKIMDomains

126 dkimdoms := []string{}

127 for _, dom := range d.m.DKIMDomains {

128 if dom != rs.VerifiedDNSDomain.Name() {

129 dkimdoms = append(dkimdoms, dom)

130 }

131 }

132 d.m.DKIMDomains = dkimdoms

133 dmarcOverrideReason = string(dmarcrpt.PolicyOverrideForwarded)

134 log.Info("forwarded message, clearing identifying signals of forwarding mail server")

135 }

136

137 assignMailbox := func(tx *bstore.Tx) error {

138 // Set message MailboxID to which mail will be delivered. Reputation is

139 // per-mailbox. If referenced mailbox is not found (e.g. does not yet exist), we

140 // can still determine a reputation because we also base it on outgoing

141 // messages and those are account-global.

142 mb, err := d.acc.MailboxFind(tx, mailbox)

143 if err != nil {

144 return fmt.Errorf("finding destination mailbox: %w", err)

145 }

146 if mb != nil {

147 // We want to deliver to mb.ID, but this message may be rejected and sent to the

148 // Rejects mailbox instead, with MailboxID overwritten. Record the ID in

149 // MailboxDestinedID too. If the message is later moved out of the Rejects mailbox,

150 // we'll adjust the MailboxOrigID so it gets taken into account during reputation

151 // calculating in future deliveries. If we end up delivering to the intended

152 // mailbox (i.e. not rejecting), MailboxDestinedID is cleared during delivery so we

153 // don't store it unnecessarily.

154 d.m.MailboxID = mb.ID

155 d.m.MailboxDestinedID = mb.ID

156 } else {

157 log.Debug("mailbox not found in database", mlog.Field("mailbox", mailbox))

158 }

159 return nil

160 }

161

162 reject := func(code int, secode string, errmsg string, err error, reason string) analysis {

163 // We may have set MailboxDestinedID below already while we had a transaction. If

164 // not, do it now. This makes it possible to use the per-mailbox reputation when a

165 // user moves the message out of the Rejects mailbox to the intended mailbox

166 // (typically Inbox).

167 if d.m.MailboxDestinedID == 0 {

168 var mberr error

169 d.acc.WithRLock(func() {

170 mberr = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {

171 return assignMailbox(tx)

172 })

173 })

174 if mberr != nil {

175 return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError, dmarcOverrideReason, headers}

176 }

177 d.m.MailboxID = 0 // We plan to reject, no need to set intended MailboxID.

178 }

179

180 accept := false

181 if rs != nil && rs.AcceptRejectsToMailbox != "" {

182 accept = true

183 mailbox = rs.AcceptRejectsToMailbox

184 d.m.IsReject = true

185 // Don't draw attention, but don't go so far as to mark as junk.

186 d.m.Seen = true

187 log.Info("accepting reject to configured mailbox due to ruleset")

188 }

189 return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason, dmarcOverrideReason, headers}

190 }

191

192 if d.dmarcUse && d.dmarcResult.Reject {

193 return reject(smtp.C550MailboxUnavail, smtp.SePol7MultiAuthFails26, "rejecting per dmarc policy", nil, reasonDMARCPolicy)

194 }

195 // todo: should we also reject messages that have a dmarc pass but an spf record "v=spf1 -all"? suggested by m3aawg best practices.

196

197 // If destination is the DMARC reporting mailbox, do additional checks and keep

198 // track of the report. We'll check reputation, defaulting to accept.

199 var dmarcReport *dmarcrpt.Feedback

200 if d.rcptAcc.destination.DMARCReports {

201 // Messages with DMARC aggregate reports must have a DMARC pass. ../rfc/7489:1866

202 if d.dmarcResult.Status != dmarc.StatusPass {

203 log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report")

204 headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n"

205 } else if report, err := dmarcrpt.ParseMessageReport(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {

206 log.Infox("parsing dmarc aggregate report", err)

207 headers += "X-Mox-DMARCReport-Error: could not parse report\r\n"

208 } else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {

209 log.Infox("parsing domain in dmarc aggregate report", err)

210 headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n"

211 } else if _, ok := mox.Conf.Domain(d); !ok {

212 log.Info("dmarc aggregate report for domain not configured, ignoring", mlog.Field("domain", d))

213 headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n"

214 } else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {

215 log.Info("dmarc aggregate report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))

216 headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n"

217 } else {

218 dmarcReport = report

219 }

220 }

221

222 // Similar to DMARC reporting, we check for the required DKIM. We'll check

223 // reputation, defaulting to accept.

224 var tlsReport *tlsrpt.Report

225 if d.rcptAcc.destination.HostTLSReports || d.rcptAcc.destination.DomainTLSReports {

226 matchesDomain := func(sigDomain dns.Domain) bool {

227 // RFC seems to require exact DKIM domain match with submitt and message From, we

228 // also allow msgFrom to be subdomain. ../rfc/8460:322

229 return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, sigDomain)

230 }

231 // Valid DKIM signature for domain must be present. We take "valid" to assume

232 // "passing", not "syntactically valid". We also check for "tlsrpt" as service.

233 // This check is optional, but if anyone goes through the trouble to explicitly

234 // list allowed services, they would be surprised to see them ignored.

235 // ../rfc/8460:320

236 ok := false

237 for _, r := range d.dkimResults {

238 // The record should have an allowed service "tlsrpt". The RFC mentions it as if

239 // the service must be specified explicitly, but the default allowed services for a

240 // DKIM record are "*", which includes "tlsrpt". Unless a DKIM record explicitly

241 // specifies services (e.g. s=email), a record will work for TLS reports. The DKIM

242 // records seen used for TLS reporting in the wild don't explicitly set "s" for

243 // services.

244 // ../rfc/8460:326

245 if r.Status == dkim.StatusPass && matchesDomain(r.Sig.Domain) && r.Sig.Length < 0 && r.Record.ServiceAllowed("tlsrpt") {

246 ok = true

247 break

248 }

249 }

250

251 if !ok {

252 log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report")

253 headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n"

254 } else if report, err := tlsrpt.ParseMessage(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {

255 log.Infox("parsing tls report", err)

256 headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n"

257 } else {

258 var known bool

259 for _, p := range report.Policies {

260 log.Info("tlsrpt policy domain", mlog.Field("domain", p.Policy.Domain))

261 if d, err := dns.ParseDomain(p.Policy.Domain); err != nil {

262 log.Infox("parsing domain in tls report", err)

263 } else if _, ok := mox.Conf.Domain(d); ok || d == mox.Conf.Static.HostnameDomain {

264 known = true

265 break

266 }

267 }

268 if !known {

269 log.Info("tls report without one of configured domains, ignoring")

270 headers += "X-Mox-TLSReport-Error: report for unknown domain\r\n"

271 } else {

272 tlsReport = report

273 }

274 }

275 }

276

277 // Determine if message is acceptable based on DMARC domain, DKIM identities, or

278 // host-based reputation.

279 var isjunk *bool

280 var conclusive bool

281 var method reputationMethod

282 var reason string

283 var err error

284 d.acc.WithRLock(func() {

285 err = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {

286 if err := assignMailbox(tx); err != nil {

287 return err

288 }

289

290 isjunk, conclusive, method, err = reputation(tx, log, d.m)

291 reason = string(method)

292 return err

293 })

294 })

295 if err != nil {

296 log.Infox("determining reputation", err, mlog.Field("message", d.m))

297 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError)

298 }

299 log.Info("reputation analyzed", mlog.Field("conclusive", conclusive), mlog.Field("isjunk", isjunk), mlog.Field("method", string(method)))

300 if conclusive {

301 if !*isjunk {

302 return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

303 }

304 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, string(method))

305 } else if dmarcReport != nil || tlsReport != nil {

306 log.Info("accepting message with dmarc aggregate report or tls report without reputation")

307 return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

308 }

309 // If there was no previous message from sender or its domain, and we have an SPF

310 // (soft)fail, reject the message.

311 switch method {

312 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:

313 switch d.m.MailFromValidation {

314 case store.ValidationFail, store.ValidationSoftfail:

315 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonSPFPolicy)

316 }

317 }

318

319 // Senders without reputation and without iprev pass, are likely spam.

320 var suspiciousIPrevFail bool

321 switch method {

322 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:

323 suspiciousIPrevFail = d.iprevStatus != iprev.StatusPass

324 }

325

326 // With already a mild junk signal, an iprev fail on top is enough to reject.

327 if suspiciousIPrevFail && isjunk != nil && *isjunk {

328 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonIPrev)

329 }

330

331 var subjectpassKey string

332 conf, _ := d.acc.Conf()

333 if conf.SubjectPass.Period > 0 {

334 subjectpassKey, err = d.acc.Subjectpass(d.rcptAcc.canonicalAddress)

335 if err != nil {

336 log.Errorx("get key for verifying subject token", err)

337 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError)

338 }

339 err = subjectpass.Verify(log, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period)

340 pass := err == nil

341 log.Infox("pass by subject token", err, mlog.Field("pass", pass))

342 if pass {

343 return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

344 }

345 }

346

347 reason = reasonNoBadSignals

348 accept := true

349 var junkSubjectpass bool

350 f, jf, err := d.acc.OpenJunkFilter(ctx, log)

351 if err == nil {

352 defer func() {

353 err := f.Close()

354 log.Check(err, "closing junkfilter")

355 }()

356 contentProb, _, _, _, err := f.ClassifyMessageReader(ctx, store.FileMsgReader(d.m.MsgPrefix, d.dataFile), d.m.Size)

357 if err != nil {

358 log.Errorx("testing for spam", err)

359 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkClassifyError)

360 }

361 // todo: if isjunk is not nil (i.e. there was inconclusive reputation), use it in the probability calculation. give reputation a score of 0.25 or .75 perhaps?

362 // todo: if there aren't enough historic messages, we should just let messages in.

363 // todo: we could require nham and nspam to be above a certain number when there were plenty of words in the message, and in the database. can indicate a spammer is misspelling words. however, it can also mean a message in a different language/script...

364

365 // If we don't accept, we may still respond with a "subjectpass" hint below.

366 // We add some jitter to the threshold we use. So we don't act as too easy an

367 // oracle for words that are a strong indicator of haminess.

368 // todo: we should rate-limit uses of the junkfilter.

369 jitter := (jitterRand.Float64() - 0.5) / 10

370 threshold := jf.Threshold + jitter

371

372 // With an iprev fail, we set a higher bar for content.

373 reason = reasonJunkContent

374 if suspiciousIPrevFail && threshold > 0.25 {

375 threshold = 0.25

376 log.Info("setting junk threshold due to iprev fail", mlog.Field("threshold", 0.25))

377 reason = reasonJunkContentStrict

378 }

379 accept = contentProb <= threshold

380 junkSubjectpass = contentProb < threshold-0.2

381 log.Info("content analyzed", mlog.Field("accept", accept), mlog.Field("contentprob", contentProb), mlog.Field("subjectpass", junkSubjectpass))

382 } else if err != store.ErrNoJunkFilter {

383 log.Errorx("open junkfilter", err)

384 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError)

385 }

386

387 // If content looks good, we'll still look at DNS block lists for a reason to

388 // reject. We normally won't get here if we've communicated with this sender

389 // before.

390 var dnsblocklisted bool

391 if accept {

392 blocked := func(zone dns.Domain) bool {

393 dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second)

394 defer dnsblcancel()

395 if !checkDNSBLHealth(dnsblctx, resolver, zone) {

396 log.Info("dnsbl not healthy, skipping", mlog.Field("zone", zone))

397 return false

398 }

399

400 status, expl, err := dnsbl.Lookup(dnsblctx, resolver, zone, net.ParseIP(d.m.RemoteIP))

401 dnsblcancel()

402 if status == dnsbl.StatusFail {

403 log.Info("rejecting due to listing in dnsbl", mlog.Field("zone", zone), mlog.Field("explanation", expl))

404 return true

405 } else if err != nil {

406 log.Infox("dnsbl lookup", err, mlog.Field("zone", zone), mlog.Field("status", status))

407 }

408 return false

409 }

410

411 // Note: We don't check in parallel, we are in no hurry to accept possible spam.

412 for _, zone := range d.dnsBLs {

413 if blocked(zone) {

414 accept = false

415 dnsblocklisted = true

416 reason = reasonDNSBlocklisted

417 break

418 }

419 }

420 }

421

422 if accept {

423 return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

424 }

425

426 if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {

427 log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation")

428 pass := subjectpass.Generate(d.msgFrom, []byte(subjectpassKey), time.Now())

429 return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass)

430 }

431

432 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reason)

433}

434