1package store

2

3import (

4 "os"

5 "path/filepath"

6 "reflect"

7 "strings"

8 "testing"

9 "time"

10

11 "github.com/mjl-/bstore"

12

13 "github.com/mjl-/mox/mlog"

14 "github.com/mjl-/mox/mox-"

15)

16

17func TestThreadingUpgrade(t *testing.T) {

18 log := mlog.New("store", nil)

19 os.RemoveAll("../testdata/store/data")

20 mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")

21 mox.MustLoadConfig(true, false)

22 acc, err := OpenAccount(log, "mjl")

23 tcheck(t, err, "open account")

24 defer func() {

25 err = acc.Close()

26 tcheck(t, err, "closing account")

27 }()

28 defer Switchboard()()

29

30 // New account already has threading. Add some messages, check the threading.

31 deliver := func(recv time.Time, s string, expThreadID int64) Message {

32 t.Helper()

33 f, err := CreateMessageTemp(log, "account-test")

34 tcheck(t, err, "temp file")

35 defer os.Remove(f.Name())

36 defer f.Close()

37

38 s = strings.ReplaceAll(s, "\n", "\r\n")

39 m := Message{

40 Size: int64(len(s)),

41 MsgPrefix: []byte(s),

42 Received: recv,

43 }

44 err = acc.DeliverMailbox(log, "Inbox", &m, f)

45 tcheck(t, err, "deliver")

46 if expThreadID == 0 {

47 expThreadID = m.ID

48 }

49 if m.ThreadID != expThreadID {

50 t.Fatalf("got threadid %d, expected %d", m.ThreadID, expThreadID)

51 }

52 return m

53 }

54

55 now := time.Now()

56

57 m0 := deliver(now, "Message-ID: <m0@localhost>\nSubject: test1\n\ntest\n", 0)

58 m1 := deliver(now, "Message-ID: <m1@localhost>\nReferences: <m0@localhost>\nSubject: test1\n\ntest\n", m0.ID) // References.

59 m2 := deliver(now, "Message-ID: <m2@localhost>\nReferences: <m0@localhost>\nSubject: other\n\ntest\n", 0) // References, but different subject.

60 m3 := deliver(now, "Message-ID: <m3@localhost>\nIn-Reply-To: <m0@localhost>\nSubject: test1\n\ntest\n", m0.ID) // In-Reply-To.

61 m4 := deliver(now, "Message-ID: <m4@localhost>\nSubject: re: test1\n\ntest\n", m0.ID) // Subject.

62 m5 := deliver(now, "Message-ID: <m5@localhost>\nSubject: test1 (fwd)\n\ntest\n", m0.ID) // Subject.

63 m6 := deliver(now, "Message-ID: <m6@localhost>\nSubject: [fwd: test1]\n\ntest\n", m0.ID) // Subject.

64 m7 := deliver(now, "Message-ID: <m7@localhost>\nSubject: test1\n\ntest\n", 0) // Only subject, but not a response.

65

66 // Thread with a cyclic head, a self-referencing message.

67 c1 := deliver(now, "Message-ID: <c1@localhost>\nReferences: <c2@localhost>\nSubject: cycle0\n\ntest\n", 0) // Head cycle with m8.

68 c2 := deliver(now, "Message-ID: <c2@localhost>\nReferences: <c1@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Head cycle with c1.

69 c3 := deliver(now, "Message-ID: <c3@localhost>\nReferences: <c1@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Connected to one of the cycle elements.

70 c4 := deliver(now, "Message-ID: <c4@localhost>\nReferences: <c2@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Connected to other cycle element.

71 c5 := deliver(now, "Message-ID: <c5@localhost>\nReferences: <c4@localhost>\nSubject: cycle0\n\ntest\n", c1.ID)

72 c5b := deliver(now, "Message-ID: <c5@localhost>\nReferences: <c4@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Duplicate, e.g. Sent item, internal cycle during upgrade.

73 c6 := deliver(now, "Message-ID: <c6@localhost>\nReferences: <c5@localhost>\nSubject: cycle0\n\ntest\n", c1.ID)

74 c7 := deliver(now, "Message-ID: <c7@localhost>\nReferences: <c5@localhost> <c7@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Self-referencing message that also points to actual parent.

75

76 // More than 2 messages to make a cycle.

77 d0 := deliver(now, "Message-ID: <d0@localhost>\nReferences: <d2@localhost>\nSubject: cycle1\n\ntest\n", 0)

78 d1 := deliver(now, "Message-ID: <d1@localhost>\nReferences: <d0@localhost>\nSubject: cycle1\n\ntest\n", d0.ID)

79 d2 := deliver(now, "Message-ID: <d2@localhost>\nReferences: <d1@localhost>\nSubject: cycle1\n\ntest\n", d0.ID)

80

81 // Cycle with messages delivered later. During import/upgrade, they will all be one thread.

82 e0 := deliver(now, "Message-ID: <e0@localhost>\nReferences: <e1@localhost>\nSubject: cycle2\n\ntest\n", 0)

83 e1 := deliver(now, "Message-ID: <e1@localhost>\nReferences: <e2@localhost>\nSubject: cycle2\n\ntest\n", 0)

84 e2 := deliver(now, "Message-ID: <e2@localhost>\nReferences: <e0@localhost>\nSubject: cycle2\n\ntest\n", e0.ID)

85

86 // Three messages in a cycle (f1, f2, f3), with one with an additional ancestor (f4) which is ignored due to the cycle. Has different threads during import.

87 f0 := deliver(now, "Message-ID: <f0@localhost>\nSubject: cycle3\n\ntest\n", 0)

88 f1 := deliver(now, "Message-ID: <f1@localhost>\nReferences: <f0@localhost> <f2@localhost>\nSubject: cycle3\n\ntest\n", f0.ID)

89 f2 := deliver(now, "Message-ID: <f2@localhost>\nReferences: <f3@localhost>\nSubject: cycle3\n\ntest\n", 0)

90 f3 := deliver(now, "Message-ID: <f3@localhost>\nReferences: <f1@localhost>\nSubject: cycle3\n\ntest\n", f0.ID)

91

92 // Duplicate single message (no larger thread).

93 g0 := deliver(now, "Message-ID: <g0@localhost>\nSubject: dup\n\ntest\n", 0)

94 g0b := deliver(now, "Message-ID: <g0@localhost>\nSubject: dup\n\ntest\n", g0.ID)

95

96 // Duplicate message with a child message.

97 h0 := deliver(now, "Message-ID: <h0@localhost>\nSubject: dup2\n\ntest\n", 0)

98 h0b := deliver(now, "Message-ID: <h0@localhost>\nSubject: dup2\n\ntest\n", h0.ID)

99 h1 := deliver(now, "Message-ID: <h1@localhost>\nReferences: <h0@localhost>\nSubject: dup2\n\ntest\n", h0.ID)

100

101 // Message has itself as reference.

102 s0 := deliver(now, "Message-ID: <s0@localhost>\nReferences: <s0@localhost>\nSubject: self-referencing message\n\ntest\n", 0)

103

104 // Message with \0 in subject, should get an empty base subject.

105 b0 := deliver(now, "Message-ID: <b0@localhost>\nSubject: bad\u0000subject\n\ntest\n", 0)

106 b1 := deliver(now, "Message-ID: <b1@localhost>\nSubject: bad\u0000subject\n\ntest\n", 0) // Not matched.

107

108 // Interleaved duplicate threaded messages. First child, then parent, then duplicate parent, then duplicat child again.

109 i0 := deliver(now, "Message-ID: <i0@localhost>\nReferences: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", 0)

110 i1 := deliver(now, "Message-ID: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", 0)

111 i2 := deliver(now, "Message-ID: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", i1.ID)

112 i3 := deliver(now, "Message-ID: <i0@localhost>\nReferences: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", i0.ID)

113

114 j0 := deliver(now, "Message-ID: <j0@localhost>\nReferences: <>\nSubject: empty id in references\n\ntest\n", 0)

115

116 dbpath := acc.DBPath

117 err = acc.Close()

118 tcheck(t, err, "close account")

119

120 // Now clear the threading upgrade, and the threading fields and close the account.

121 // We open the database file directly, so we don't trigger the consistency checker.

122 db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)

123 err = db.Write(ctxbg, func(tx *bstore.Tx) error {

124 up := Upgrade{ID: 1}

125 err := tx.Delete(&up)

126 tcheck(t, err, "delete upgrade")

127

128 q := bstore.QueryTx[Message](tx)

129 _, err = q.UpdateFields(map[string]any{

130 "MessageID": "",

131 "SubjectBase": "",

132 "ThreadID": int64(0),

133 "ThreadParentIDs": []int64(nil),

134 "ThreadMissingLink": false,

135 })

136 return err

137 })

138 tcheck(t, err, "reset threading fields")

139 err = db.Close()

140 tcheck(t, err, "closing db")

141

142 // Open the account again, that should get the account upgraded. Wait for upgrade to finish.

143 acc, err = OpenAccount(log, "mjl")

144 tcheck(t, err, "open account")

145 err = acc.ThreadingWait(log)

146 tcheck(t, err, "wait for threading")

147

148 check := func(id int64, expThreadID int64, expParentIDs []int64, expMissingLink bool) {

149 t.Helper()

150

151 m := Message{ID: id}

152 err := acc.DB.Get(ctxbg, &m)

153 tcheck(t, err, "get message")

154 if m.ThreadID != expThreadID || !reflect.DeepEqual(m.ThreadParentIDs, expParentIDs) || m.ThreadMissingLink != expMissingLink {

155 t.Fatalf("got thread id %d, parent ids %v, missing link %v, expected %d %v %v", m.ThreadID, m.ThreadParentIDs, m.ThreadMissingLink, expThreadID, expParentIDs, expMissingLink)

156 }

157 }

158

159 parents0 := []int64{m0.ID}

160 check(m0.ID, m0.ID, nil, false)

161 check(m1.ID, m0.ID, parents0, false)

162 check(m2.ID, m2.ID, nil, true)

163 check(m3.ID, m0.ID, parents0, false)

164 check(m4.ID, m0.ID, parents0, true)

165 check(m5.ID, m0.ID, parents0, true)

166 check(m6.ID, m0.ID, parents0, true)

167 check(m7.ID, m7.ID, nil, false)

168

169 check(c1.ID, c1.ID, nil, true) // Head of cycle, hence missing link

170 check(c2.ID, c1.ID, []int64{c1.ID}, false)

171 check(c3.ID, c1.ID, []int64{c1.ID}, false)

172 check(c4.ID, c1.ID, []int64{c2.ID, c1.ID}, false)

173 check(c5.ID, c1.ID, []int64{c4.ID, c2.ID, c1.ID}, false)

174 check(c5b.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, true)

175 check(c6.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, false)

176 check(c7.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, true)

177

178 check(d0.ID, d0.ID, nil, true)

179 check(d1.ID, d0.ID, []int64{d0.ID}, false)

180 check(d2.ID, d0.ID, []int64{d1.ID, d0.ID}, false)

181

182 check(e0.ID, e0.ID, nil, true)

183 check(e1.ID, e0.ID, []int64{e2.ID, e0.ID}, false)

184 check(e2.ID, e0.ID, []int64{e0.ID}, false)

185

186 check(f0.ID, f0.ID, nil, false)

187 check(f1.ID, f1.ID, nil, true)

188 check(f2.ID, f1.ID, []int64{f3.ID, f1.ID}, false)

189 check(f3.ID, f1.ID, []int64{f1.ID}, false)

190

191 check(g0.ID, g0.ID, nil, false)

192 check(g0b.ID, g0.ID, []int64{g0.ID}, true)

193

194 check(h0.ID, h0.ID, nil, false)

195 check(h0b.ID, h0.ID, []int64{h0.ID}, true)

196 check(h1.ID, h0.ID, []int64{h0.ID}, false)

197

198 check(s0.ID, s0.ID, nil, true)

199

200 check(b0.ID, b0.ID, nil, false)

201 check(b1.ID, b1.ID, nil, false)

202

203 check(i0.ID, i1.ID, []int64{i1.ID}, false)

204 check(i1.ID, i1.ID, nil, false)

205 check(i2.ID, i1.ID, []int64{i1.ID}, true)

206 check(i3.ID, i1.ID, []int64{i0.ID, i1.ID}, true)

207

208 check(j0.ID, j0.ID, nil, false)

209}

210