1// Package mtastsdb stores MTA-STS policies for later use.
2//
3// An MTA-STS policy can specify how long it may be cached. By storing a
4// policy, it does not have to be fetched again during email delivery, which
5// makes it harder for attackers to intervene.
6package mtastsdb
7
8import (
9 "context"
10 "errors"
11 "fmt"
12 "os"
13 "path/filepath"
14 "sync"
15 "time"
16
17 "github.com/prometheus/client_golang/prometheus"
18 "github.com/prometheus/client_golang/prometheus/promauto"
19
20 "github.com/mjl-/bstore"
21
22 "github.com/mjl-/mox/dns"
23 "github.com/mjl-/mox/mlog"
24 "github.com/mjl-/mox/mox-"
25 "github.com/mjl-/mox/mtasts"
26)
27
28var xlog = mlog.New("mtastsdb")
29
30var (
31 metricGet = promauto.NewCounterVec(
32 prometheus.CounterOpts{
33 Name: "mox_mtastsdb_get_total",
34 Help: "Number of Get by result.",
35 },
36 []string{"result"},
37 )
38)
39
40var timeNow = time.Now // Tests override this.
41
42// PolicyRecord is a cached policy or absence of a policy.
43type PolicyRecord struct {
44 Domain string // Domain name, with unicode characters.
45 Inserted time.Time `bstore:"default now"`
46 ValidEnd time.Time
47 LastUpdate time.Time // Policies are refreshed on use and periodically.
48 LastUse time.Time `bstore:"index"`
49 Backoff bool
50 RecordID string // As retrieved from DNS.
51 mtasts.Policy // As retrieved from the well-known HTTPS url.
52}
53
54var (
55 // No valid non-expired policy in database.
56 ErrNotFound = errors.New("mtastsdb: policy not found")
57
58 // Indicates an MTA-STS TXT record was fetched recently, but fetching the policy
59 // failed and should not yet be retried.
60 ErrBackoff = errors.New("mtastsdb: policy fetch failed recently")
61)
62
63var DBTypes = []any{PolicyRecord{}} // Types stored in DB.
64var DB *bstore.DB // Exported for backups.
65var mutex sync.Mutex
66
67func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
68 mutex.Lock()
69 defer mutex.Unlock()
70 if DB == nil {
71 p := mox.DataDirPath("mtasts.db")
72 os.MkdirAll(filepath.Dir(p), 0770)
73 db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
74 if err != nil {
75 return nil, err
76 }
77 DB = db
78 }
79 return DB, nil
80}
81
82// Init opens the database and starts a goroutine that refreshes policies in
83// the database, and keeps doing so periodically.
84func Init(refresher bool) error {
85 _, err := database(mox.Shutdown)
86 if err != nil {
87 return err
88 }
89
90 if refresher {
91 // todo: allow us to shut down cleanly?
92 go refresh()
93 }
94
95 return nil
96}
97
98// Close closes the database.
99func Close() {
100 mutex.Lock()
101 defer mutex.Unlock()
102 if DB != nil {
103 err := DB.Close()
104 xlog.Check(err, "closing database")
105 DB = nil
106 }
107}
108
109// Lookup looks up a policy for the domain in the database.
110//
111// Only non-expired records are returned.
112func lookup(ctx context.Context, domain dns.Domain) (*PolicyRecord, error) {
113 log := xlog.WithContext(ctx)
114 db, err := database(ctx)
115 if err != nil {
116 return nil, err
117 }
118
119 if domain.IsZero() {
120 return nil, fmt.Errorf("empty domain")
121 }
122 now := timeNow()
123 q := bstore.QueryDB[PolicyRecord](ctx, db)
124 q.FilterNonzero(PolicyRecord{Domain: domain.Name()})
125 q.FilterGreater("ValidEnd", now)
126 pr, err := q.Get()
127 if err == bstore.ErrAbsent {
128 return nil, ErrNotFound
129 } else if err != nil {
130 return nil, err
131 }
132
133 pr.LastUse = now
134 if err := db.Update(ctx, &pr); err != nil {
135 log.Errorx("marking cached mta-sts policy as used in database", err)
136 }
137 if pr.Backoff {
138 return nil, ErrBackoff
139 }
140 return &pr, nil
141}
142
143// Upsert adds the policy to the database, overwriting an existing policy for the domain.
144// Policy can be nil, indicating a failure to fetch the policy.
145func Upsert(ctx context.Context, domain dns.Domain, recordID string, policy *mtasts.Policy) error {
146 db, err := database(ctx)
147 if err != nil {
148 return err
149 }
150
151 return db.Write(ctx, func(tx *bstore.Tx) error {
152 pr := PolicyRecord{Domain: domain.Name()}
153 err := tx.Get(&pr)
154 if err != nil && err != bstore.ErrAbsent {
155 return err
156 }
157
158 now := timeNow()
159
160 var p mtasts.Policy
161 if policy != nil {
162 p = *policy
163 } else {
164 // ../rfc/8461:552
165 p.Mode = mtasts.ModeNone
166 p.MaxAgeSeconds = 5 * 60
167 }
168 backoff := policy == nil
169 validEnd := now.Add(time.Duration(p.MaxAgeSeconds) * time.Second)
170
171 if err == bstore.ErrAbsent {
172 pr = PolicyRecord{domain.Name(), now, validEnd, now, now, backoff, recordID, p}
173 return tx.Insert(&pr)
174 }
175
176 pr.ValidEnd = validEnd
177 pr.LastUpdate = now
178 pr.LastUse = now
179 pr.Backoff = backoff
180 pr.RecordID = recordID
181 pr.Policy = p
182 return tx.Update(&pr)
183 })
184}
185
186// PolicyRecords returns all policies in the database, sorted descending by last
187// use, domain.
188func PolicyRecords(ctx context.Context) ([]PolicyRecord, error) {
189 db, err := database(ctx)
190 if err != nil {
191 return nil, err
192 }
193 return bstore.QueryDB[PolicyRecord](ctx, db).SortDesc("LastUse", "Domain").List()
194}
195
196// Get retrieves an MTA-STS policy for domain and whether it is fresh.
197//
198// If an error is returned, it should be considered a transient error, e.g. a
199// temporary DNS lookup failure.
200//
201// The returned policy can be nil also when there is no error. In this case, the
202// domain does not implement MTA-STS.
203//
204// If a policy is present in the local database, it is refreshed if needed. If no
205// policy is present for the domain, an attempt is made to fetch the policy and
206// store it in the local database.
207//
208// Some errors are logged but not otherwise returned, e.g. if a new policy is
209// supposedly published but could not be retrieved.
210func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy *mtasts.Policy, fresh bool, err error) {
211 log := xlog.WithContext(ctx)
212 defer func() {
213 result := "ok"
214 if err != nil && errors.Is(err, ErrBackoff) {
215 result = "backoff"
216 } else if err != nil && errors.Is(err, ErrNotFound) {
217 result = "notfound"
218 } else if err != nil {
219 result = "error"
220 }
221 metricGet.WithLabelValues(result).Inc()
222 log.Debugx("mtastsdb get result", err, mlog.Field("domain", domain), mlog.Field("fresh", fresh))
223 }()
224
225 cachedPolicy, err := lookup(ctx, domain)
226 if err != nil && errors.Is(err, ErrNotFound) {
227 // We don't have a policy for this domain, not even a record that we tried recently
228 // and should backoff. So attempt to fetch policy.
229 nctx, cancel := context.WithTimeout(ctx, time.Minute)
230 defer cancel()
231 record, p, err := mtasts.Get(nctx, resolver, domain)
232 if err != nil {
233 switch {
234 case errors.Is(err, mtasts.ErrNoRecord) || errors.Is(err, mtasts.ErrMultipleRecords) || errors.Is(err, mtasts.ErrRecordSyntax) || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax):
235 // Remote is not doing MTA-STS, continue below. ../rfc/8461:333 ../rfc/8461:574
236 log.Debugx("interpreting mtasts error to mean remote is not doing mta-sts", err)
237 default:
238 // Interpret as temporary error, e.g. mtasts.ErrDNS, try again later.
239 return nil, false, fmt.Errorf("lookup up mta-sts policy: %w", err)
240 }
241 }
242 // Insert policy into database. If we could not fetch the policy itself, we back
243 // off for 5 minutes. ../rfc/8461:555
244 if err == nil || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax) {
245 var recordID string
246 if record != nil {
247 recordID = record.ID
248 }
249 if err := Upsert(ctx, domain, recordID, p); err != nil {
250 log.Errorx("inserting policy into cache, continuing", err)
251 }
252 }
253 return p, true, nil
254 } else if err != nil && errors.Is(err, ErrBackoff) {
255 // ../rfc/8461:552
256 // We recently failed to fetch a policy, act as if MTA-STS is not implemented.
257 return nil, false, nil
258 } else if err != nil {
259 return nil, false, fmt.Errorf("looking up mta-sts policy in cache: %w", err)
260 }
261
262 // Policy was found in database. Check in DNS it is still fresh.
263 policy = &cachedPolicy.Policy
264 nctx, cancel := context.WithTimeout(ctx, 30*time.Second)
265 defer cancel()
266 record, _, _, err := mtasts.LookupRecord(nctx, resolver, domain)
267 if err != nil {
268 if !errors.Is(err, mtasts.ErrNoRecord) {
269 // Could be a temporary DNS or configuration error.
270 log.Errorx("checking for freshness of cached mta-sts dns txt record for domain, continuing with previously cached policy", err)
271 }
272 return policy, false, nil
273 } else if record.ID == cachedPolicy.RecordID {
274 return policy, true, nil
275 }
276
277 // New policy should be available.
278 nctx, cancel = context.WithTimeout(ctx, 30*time.Second)
279 defer cancel()
280 p, _, err := mtasts.FetchPolicy(nctx, domain)
281 if err != nil {
282 log.Errorx("fetching updated policy for domain, continuing with previously cached policy", err)
283 return policy, false, nil
284 }
285 if err := Upsert(ctx, domain, record.ID, p); err != nil {
286 log.Errorx("inserting refreshed policy into cache, continuing with fresh policy", err)
287 }
288 return p, true, nil
289}
290