1//go:generate sh -c "curl https://publicsuffix.org/list/public_suffix_list.dat >public_suffix_list.txt"
2
3// Package publicsuffix implements a public suffix list to look up the
4// organizational domain for a given host name. Organizational domains can be
5// registered, one level below a top-level domain.
6//
7// Example.com has a public suffix ".com", and example.co.uk has a public
8// suffix ".co.uk". The organizational domain of sub.example.com is
9// example.com, and the organization domain of sub.example.co.uk is
10// example.co.uk.
11package publicsuffix
12
13import (
14 "bufio"
15 "bytes"
16 "context"
17 "fmt"
18 "io"
19 "log/slog"
20 "slices"
21 "strings"
22
23 _ "embed"
24
25 "golang.org/x/net/idna"
26
27 "github.com/mjl-/mox/dns"
28 "github.com/mjl-/mox/mlog"
29)
30
31// todo: automatically fetch new lists periodically? compare it with the old one. refuse it if it changed too much, especially if it contains far fewer entries than before.
32
33// Labels map from utf8 labels to labels for subdomains.
34// The end is marked with an empty string as label.
35type labels map[string]labels
36
37// List is a public suffix list.
38type List struct {
39 includes, excludes labels
40}
41
42var publicsuffixList List
43
44//go:embed public_suffix_list.txt
45var publicsuffixData []byte
46
47func init() {
48 log := mlog.New("publicsuffix", nil)
49 l, err := ParseList(log.Logger, bytes.NewReader(publicsuffixData))
50 if err != nil {
51 log.Fatalx("parsing public suffix list", err)
52 }
53 publicsuffixList = l
54}
55
56// ParseList parses a public suffix list.
57// Only the "ICANN DOMAINS" are used.
58func ParseList(elog *slog.Logger, r io.Reader) (List, error) {
59 log := mlog.New("publicsuffix", elog)
60
61 list := List{labels{}, labels{}}
62 br := bufio.NewReader(r)
63
64 // Only use ICANN domains. ../rfc/7489-eid6729
65 var icannDomains bool
66 for {
67 line, err := br.ReadString('\n')
68 if line != "" {
69 line = strings.TrimSpace(line)
70 if strings.HasPrefix(line, "// ===BEGIN ICANN DOMAINS===") {
71 icannDomains = true
72 continue
73 } else if strings.HasPrefix(line, "// ===END ICANN DOMAINS===") {
74 icannDomains = false
75 continue
76 } else if line == "" || strings.HasPrefix(line, "//") || !icannDomains {
77 continue
78 }
79 l := list.includes
80 var t []string
81 oline := line
82 if strings.HasPrefix(line, "!") {
83 line = line[1:]
84 l = list.excludes
85 t = strings.Split(line, ".")
86 if len(t) == 1 {
87 log.Print("exclude rule with single label, skipping", slog.String("line", oline))
88 continue
89 }
90 } else {
91 t = strings.Split(line, ".")
92 }
93 for i, w := range slices.Backward(t) {
94
95 if w == "" {
96 log.Print("empty label in rule, skipping", slog.String("line", oline))
97 break
98 }
99 if w != "" && w != "*" {
100 w, err = idna.Lookup.ToUnicode(w)
101 if err != nil {
102 log.Printx("invalid label, skipping", err, slog.String("line", oline))
103 }
104 }
105 m, ok := l[w]
106 if ok {
107 if _, dup := m[""]; i == 0 && dup {
108 log.Print("duplicate rule", slog.String("line", oline))
109 }
110 l = m
111 } else {
112 m = labels{}
113 l[w] = m
114 l = m
115 }
116 }
117 l[""] = nil // Mark end.
118 }
119 if err == io.EOF {
120 break
121 }
122 if err != nil {
123 return List{}, fmt.Errorf("reading public suffix list: %w", err)
124 }
125 }
126 return list, nil
127}
128
129// Lookup calls Lookup on the builtin public suffix list, from
130// https://publicsuffix.org/list/.
131func Lookup(ctx context.Context, elog *slog.Logger, domain dns.Domain) (orgDomain dns.Domain) {
132 return publicsuffixList.Lookup(ctx, elog, domain)
133}
134
135// Lookup returns the organizational domain. If domain is an organizational
136// domain, or higher-level, the same domain is returned.
137func (l List) Lookup(ctx context.Context, elog *slog.Logger, domain dns.Domain) (orgDomain dns.Domain) {
138 log := mlog.New("publicsuffix", elog)
139 defer func() {
140 log.Debug("publicsuffix lookup result", slog.Any("reqdom", domain), slog.Any("orgdom", orgDomain))
141 }()
142
143 t := strings.Split(domain.Name(), ".")
144
145 var n int
146 if nexcl, ok := match(l.excludes, t); ok {
147 n = nexcl
148 } else if nincl, ok := match(l.includes, t); ok {
149 n = nincl + 1
150 } else {
151 n = 2
152 }
153 if len(t) < n {
154 return domain
155 }
156 name := strings.Join(t[len(t)-n:], ".")
157 if isASCII(name) {
158 return dns.Domain{ASCII: name}
159 }
160 t = strings.Split(domain.ASCII, ".")
161 ascii := strings.Join(t[len(t)-n:], ".")
162 return dns.Domain{ASCII: ascii, Unicode: name}
163}
164
165func isASCII(s string) bool {
166 for _, c := range s {
167 if c >= 0x80 {
168 return false
169 }
170 }
171 return true
172}
173
174func match(l labels, t []string) (int, bool) {
175 if len(t) == 0 {
176 _, ok := l[""]
177 return 0, ok
178 }
179 s := t[len(t)-1]
180 t = t[:len(t)-1]
181 n := 0
182 if m, mok := l[s]; mok {
183 if nn, sok := match(m, t); sok {
184 n = 1 + nn
185 }
186 }
187 if m, mok := l["*"]; mok {
188 if nn, sok := match(m, t); sok && nn >= n {
189 n = 1 + nn
190 }
191 }
192 _, mok := l[""]
193 return n, n > 0 || mok
194}
195