1package smtpserver

2

3import (

4 "context"

5 "fmt"

6 "net"

7 "os"

8 "time"

9

10 "github.com/mjl-/bstore"

11

12 "github.com/mjl-/mox/dkim"

13 "github.com/mjl-/mox/dmarc"

14 "github.com/mjl-/mox/dmarcrpt"

15 "github.com/mjl-/mox/dns"

16 "github.com/mjl-/mox/dnsbl"

17 "github.com/mjl-/mox/iprev"

18 "github.com/mjl-/mox/mlog"

19 "github.com/mjl-/mox/mox-"

20 "github.com/mjl-/mox/smtp"

21 "github.com/mjl-/mox/store"

22 "github.com/mjl-/mox/subjectpass"

23 "github.com/mjl-/mox/tlsrpt"

24)

25

26type delivery struct {

27 m *store.Message

28 dataFile *os.File

29 rcptAcc rcptAccount

30 acc *store.Account

31 msgFrom smtp.Address

32 dnsBLs []dns.Domain

33 dmarcUse bool

34 dmarcResult dmarc.Result

35 dkimResults []dkim.Result

36 iprevStatus iprev.Status

37}

38

39type analysis struct {

40 accept bool

41 mailbox string

42 code int

43 secode string

44 userError bool

45 errmsg string

46 err error // For our own logging, not sent to remote.

47 dmarcReport *dmarcrpt.Feedback // Validated dmarc aggregate report, not yet stored.

48 tlsReport *tlsrpt.Report // Validated TLS report, not yet stored.

49 reason string // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens.

50}

51

52const (

53 reasonListAllow = "list-allow"

54 reasonDMARCPolicy = "dmarc-policy"

55 reasonReputationError = "reputation-error"

56 reasonReporting = "reporting"

57 reasonSPFPolicy = "spf-policy"

58 reasonJunkClassifyError = "junk-classify-error"

59 reasonJunkFilterError = "junk-filter-error"

60 reasonGiveSubjectpass = "give-subjectpass"

61 reasonNoBadSignals = "no-bad-signals"

62 reasonJunkContent = "junk-content"

63 reasonJunkContentStrict = "junk-content-strict"

64 reasonDNSBlocklisted = "dns-blocklisted"

65 reasonSubjectpass = "subjectpass"

66 reasonSubjectpassError = "subjectpass-error"

67 reasonIPrev = "iprev" // No or mil junk reputation signals, and bad iprev.

68)

69

70func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delivery) analysis {

71 mailbox := d.rcptAcc.destination.Mailbox

72 if mailbox == "" {

73 mailbox = "Inbox"

74 }

75

76 // If destination mailbox has a mailing list domain (for SPF/DKIM) configured,

77 // check it for a pass.

78 rs := store.MessageRuleset(log, d.rcptAcc.destination, d.m, d.m.MsgPrefix, d.dataFile)

79 if rs != nil {

80 mailbox = rs.Mailbox

81 }

82 if rs != nil && !rs.ListAllowDNSDomain.IsZero() {

83 ld := rs.ListAllowDNSDomain

84 // todo: on temporary failures, reject temporarily?

85 if d.m.MailFromValidated && ld.Name() == d.m.MailFromDomain {

86 return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow}

87 }

88 for _, r := range d.dkimResults {

89 if r.Status == dkim.StatusPass && r.Sig.Domain == ld {

90 return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow}

91 }

92 }

93 }

94

95 // For forwarded messages, we have different junk analysis. We don't reject for

96 // failing DMARC, and we clear fields that could implicate the forwarding mail

97 // server during future classifications on incoming messages (the forwarding mail

98 // server isn't responsible for the message).

99 if rs != nil && rs.IsForward {

100 d.dmarcUse = false

101 d.m.IsForward = true

102 d.m.RemoteIPMasked1 = ""

103 d.m.RemoteIPMasked2 = ""

104 d.m.RemoteIPMasked3 = ""

105 d.m.OrigEHLODomain = d.m.EHLODomain

106 d.m.EHLODomain = ""

107 d.m.MailFromDomain = "" // Still available in MailFrom.

108 d.m.OrigDKIMDomains = d.m.DKIMDomains

109 dkimdoms := []string{}

110 for _, dom := range d.m.DKIMDomains {

111 if dom != rs.VerifiedDNSDomain.Name() {

112 dkimdoms = append(dkimdoms, dom)

113 }

114 }

115 d.m.DKIMDomains = dkimdoms

116 log.Info("forwarded message, clearing identifying signals of forwarding mail server")

117 }

118

119 assignMailbox := func(tx *bstore.Tx) error {

120 // Set message MailboxID to which mail will be delivered. Reputation is

121 // per-mailbox. If referenced mailbox is not found (e.g. does not yet exist), we

122 // can still determine a reputation because we also base it on outgoing

123 // messages and those are account-global.

124 mb, err := d.acc.MailboxFind(tx, mailbox)

125 if err != nil {

126 return fmt.Errorf("finding destination mailbox: %w", err)

127 }

128 if mb != nil {

129 // We want to deliver to mb.ID, but this message may be rejected and sent to the

130 // Rejects mailbox instead, with MailboxID overwritten. Record the ID in

131 // MailboxDestinedID too. If the message is later moved out of the Rejects mailbox,

132 // we'll adjust the MailboxOrigID so it gets taken into account during reputation

133 // calculating in future deliveries. If we end up delivering to the intended

134 // mailbox (i.e. not rejecting), MailboxDestinedID is cleared during delivery so we

135 // don't store it unnecessarily.

136 d.m.MailboxID = mb.ID

137 d.m.MailboxDestinedID = mb.ID

138 } else {

139 log.Debug("mailbox not found in database", mlog.Field("mailbox", mailbox))

140 }

141 return nil

142 }

143

144 reject := func(code int, secode string, errmsg string, err error, reason string) analysis {

145 // We may have set MailboxDestinedID below already while we had a transaction. If

146 // not, do it now. This makes it possible to use the per-mailbox reputation when a

147 // user moves the message out of the Rejects mailbox to the intended mailbox

148 // (typically Inbox).

149 if d.m.MailboxDestinedID == 0 {

150 var mberr error

151 d.acc.WithRLock(func() {

152 mberr = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {

153 return assignMailbox(tx)

154 })

155 })

156 if mberr != nil {

157 return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError}

158 }

159 d.m.MailboxID = 0 // We plan to reject, no need to set intended MailboxID.

160 }

161

162 accept := false

163 if rs != nil && rs.AcceptRejectsToMailbox != "" {

164 accept = true

165 mailbox = rs.AcceptRejectsToMailbox

166 d.m.IsReject = true

167 // Don't draw attention, but don't go so far as to mark as junk.

168 d.m.Seen = true

169 log.Info("accepting reject to configured mailbox due to ruleset")

170 }

171 return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason}

172 }

173

174 if d.dmarcUse && d.dmarcResult.Reject {

175 return reject(smtp.C550MailboxUnavail, smtp.SePol7MultiAuthFails26, "rejecting per dmarc policy", nil, reasonDMARCPolicy)

176 }

177 // todo: should we also reject messages that have a dmarc pass but an spf record "v=spf1 -all"? suggested by m3aawg best practices.

178

179 // If destination is the DMARC reporting mailbox, do additional checks and keep

180 // track of the report. We'll check reputation, defaulting to accept.

181 var dmarcReport *dmarcrpt.Feedback

182 if d.rcptAcc.destination.DMARCReports {

183 // Messages with DMARC aggregate reports must have a dmarc pass. ../rfc/7489:1866

184 if d.dmarcResult.Status != dmarc.StatusPass {

185 log.Info("received dmarc report without dmarc pass, not processing as dmarc report")

186 } else if report, err := dmarcrpt.ParseMessageReport(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {

187 log.Infox("parsing dmarc report", err)

188 } else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {

189 log.Infox("parsing domain in dmarc report", err)

190 } else if _, ok := mox.Conf.Domain(d); !ok {

191 log.Info("dmarc report for domain not configured, ignoring", mlog.Field("domain", d))

192 } else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {

193 log.Info("dmarc report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))

194 } else {

195 dmarcReport = report

196 }

197 }

198

199 // Similar to DMARC reporting, we check for the required DKIM. We'll check

200 // reputation, defaulting to accept.

201 var tlsReport *tlsrpt.Report

202 if d.rcptAcc.destination.TLSReports {

203 // Valid DKIM signature for domain must be present. We take "valid" to assume

204 // "passing", not "syntactically valid". We also check for "tlsrpt" as service.

205 // This check is optional, but if anyone goes through the trouble to explicitly

206 // list allowed services, they would be surprised to see them ignored.

207 // ../rfc/8460:320

208 ok := false

209 for _, r := range d.dkimResults {

210 if r.Status == dkim.StatusPass && r.Sig.Domain == d.msgFrom.Domain && r.Sig.Length < 0 && r.Record.ServiceAllowed("tlsrpt") {

211 ok = true

212 break

213 }

214 }

215

216 if !ok {

217 log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report")

218 } else if report, err := tlsrpt.ParseMessage(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {

219 log.Infox("parsing tls report", err)

220 } else {

221 var known bool

222 for _, p := range report.Policies {

223 log.Info("tlsrpt policy domain", mlog.Field("domain", p.Policy.Domain))

224 if d, err := dns.ParseDomain(p.Policy.Domain); err != nil {

225 log.Infox("parsing domain in tls report", err)

226 } else if _, ok := mox.Conf.Domain(d); ok {

227 known = true

228 break

229 }

230 }

231 if !known {

232 log.Info("tls report without one of configured domains, ignoring")

233 } else {

234 tlsReport = report

235 }

236 }

237 }

238

239 // Determine if message is acceptable based on DMARC domain, DKIM identities, or

240 // host-based reputation.

241 var isjunk *bool

242 var conclusive bool

243 var method reputationMethod

244 var reason string

245 var err error

246 d.acc.WithRLock(func() {

247 err = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {

248 if err := assignMailbox(tx); err != nil {

249 return err

250 }

251

252 isjunk, conclusive, method, err = reputation(tx, log, d.m)

253 reason = string(method)

254 return err

255 })

256 })

257 if err != nil {

258 log.Infox("determining reputation", err, mlog.Field("message", d.m))

259 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError)

260 }

261 log.Info("reputation analyzed", mlog.Field("conclusive", conclusive), mlog.Field("isjunk", isjunk), mlog.Field("method", string(method)))

262 if conclusive {

263 if !*isjunk {

264 return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason}

265 }

266 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, string(method))

267 } else if dmarcReport != nil || tlsReport != nil {

268 log.Info("accepting dmarc reporting or tlsrpt message without reputation")

269 return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting}

270 }

271 // If there was no previous message from sender or its domain, and we have an SPF

272 // (soft)fail, reject the message.

273 switch method {

274 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:

275 switch d.m.MailFromValidation {

276 case store.ValidationFail, store.ValidationSoftfail:

277 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonSPFPolicy)

278 }

279 }

280

281 // Senders without reputation and without iprev pass, are likely spam.

282 var suspiciousIPrevFail bool

283 switch method {

284 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:

285 suspiciousIPrevFail = d.iprevStatus != iprev.StatusPass

286 }

287

288 // With already a mild junk signal, an iprev fail on top is enough to reject.

289 if suspiciousIPrevFail && isjunk != nil && *isjunk {

290 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonIPrev)

291 }

292

293 var subjectpassKey string

294 conf, _ := d.acc.Conf()

295 if conf.SubjectPass.Period > 0 {

296 subjectpassKey, err = d.acc.Subjectpass(d.rcptAcc.canonicalAddress)

297 if err != nil {

298 log.Errorx("get key for verifying subject token", err)

299 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError)

300 }

301 err = subjectpass.Verify(log, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period)

302 pass := err == nil

303 log.Infox("pass by subject token", err, mlog.Field("pass", pass))

304 if pass {

305 return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass}

306 }

307 }

308

309 reason = reasonNoBadSignals

310 accept := true

311 var junkSubjectpass bool

312 f, jf, err := d.acc.OpenJunkFilter(ctx, log)

313 if err == nil {

314 defer func() {

315 err := f.Close()

316 log.Check(err, "closing junkfilter")

317 }()

318 contentProb, _, _, _, err := f.ClassifyMessageReader(ctx, store.FileMsgReader(d.m.MsgPrefix, d.dataFile), d.m.Size)

319 if err != nil {

320 log.Errorx("testing for spam", err)

321 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkClassifyError)

322 }

323 // todo: if isjunk is not nil (i.e. there was inconclusive reputation), use it in the probability calculation. give reputation a score of 0.25 or .75 perhaps?

324 // todo: if there aren't enough historic messages, we should just let messages in.

325 // todo: we could require nham and nspam to be above a certain number when there were plenty of words in the message, and in the database. can indicate a spammer is misspelling words. however, it can also mean a message in a different language/script...

326

327 // If we don't accept, we may still respond with a "subjectpass" hint below.

328 // We add some jitter to the threshold we use. So we don't act as too easy an

329 // oracle for words that are a strong indicator of haminess.

330 // todo: we should rate-limit uses of the junkfilter.

331 jitter := (jitterRand.Float64() - 0.5) / 10

332 threshold := jf.Threshold + jitter

333

334 // With an iprev fail, we set a higher bar for content.

335 reason = reasonJunkContent

336 if suspiciousIPrevFail && threshold > 0.25 {

337 threshold = 0.25

338 log.Info("setting junk threshold due to iprev fail", mlog.Field("threshold", 0.25))

339 reason = reasonJunkContentStrict

340 }

341 accept = contentProb <= threshold

342 junkSubjectpass = contentProb < threshold-0.2

343 log.Info("content analyzed", mlog.Field("accept", accept), mlog.Field("contentprob", contentProb), mlog.Field("subjectpass", junkSubjectpass))

344 } else if err != store.ErrNoJunkFilter {

345 log.Errorx("open junkfilter", err)

346 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError)

347 }

348

349 // If content looks good, we'll still look at DNS block lists for a reason to

350 // reject. We normally won't get here if we've communicated with this sender

351 // before.

352 var dnsblocklisted bool

353 if accept {

354 blocked := func(zone dns.Domain) bool {

355 dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second)

356 defer dnsblcancel()

357 if !checkDNSBLHealth(dnsblctx, resolver, zone) {

358 log.Info("dnsbl not healthy, skipping", mlog.Field("zone", zone))

359 return false

360 }

361

362 status, expl, err := dnsbl.Lookup(dnsblctx, resolver, zone, net.ParseIP(d.m.RemoteIP))

363 dnsblcancel()

364 if status == dnsbl.StatusFail {

365 log.Info("rejecting due to listing in dnsbl", mlog.Field("zone", zone), mlog.Field("explanation", expl))

366 return true

367 } else if err != nil {

368 log.Infox("dnsbl lookup", err, mlog.Field("zone", zone), mlog.Field("status", status))

369 }

370 return false

371 }

372

373 // Note: We don't check in parallel, we are in no hurry to accept possible spam.

374 for _, zone := range d.dnsBLs {

375 if blocked(zone) {

376 accept = false

377 dnsblocklisted = true

378 reason = reasonDNSBlocklisted

379 break

380 }

381 }

382 }

383

384 if accept {

385 return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals}

386 }

387

388 if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {

389 log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation")

390 pass := subjectpass.Generate(d.msgFrom, []byte(subjectpassKey), time.Now())

391 return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass)

392 }

393

394 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reason)

395}

396