1package imapserver

2

3import (

4 "fmt"

5 "strings"

6 "testing"

7

8 "github.com/mjl-/bstore"

9

10 "github.com/mjl-/mox/imapclient"

11 "github.com/mjl-/mox/mox-"

12 "github.com/mjl-/mox/store"

13 "slices"

14)

15

16func TestCondstore(t *testing.T) {

17 testCondstoreQresync(t, false, false)

18}

19

20func TestCondstoreUIDOnly(t *testing.T) {

21 testCondstoreQresync(t, false, true)

22}

23

24func TestQresync(t *testing.T) {

25 testCondstoreQresync(t, true, false)

26}

27

28func TestQresyncUIDOnly(t *testing.T) {

29 testCondstoreQresync(t, true, true)

30}

31

32func testCondstoreQresync(t *testing.T, qresync, uidonly bool) {

33 defer mockUIDValidity()()

34 tc := start(t, uidonly)

35 defer tc.close()

36

37 // todo: check whether marking \seen will cause modseq to be returned in case of qresync.

38

39 // Check basic requirements of CONDSTORE.

40

41 capability := imapclient.CapCondstore

42 if qresync {

43 capability = imapclient.CapQresync

44 }

45

46 tc.login("mjl@mox.example", password0)

47 tc.client.Enable(capability)

48 tc.transactf("ok", "Select inbox")

49 tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeHighestModSeq(2), Text: "x"})

50

51 // First some tests without any messages.

52

53 tc.transactf("ok", "Status inbox (Highestmodseq)")

54 tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusHighestModSeq: 2}})

55

56 // No messages, no matches.

57 tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 12345)")

58 tc.xuntagged()

59

60 // Also no messages with modseq 1, which we internally turn into modseq 0.

61 tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1)")

62 tc.xuntagged()

63

64 // Also try with modseq attribute.

65 tc.transactf("ok", "Uid Fetch 1:* (Flags Modseq) (Changedsince 1)")

66 tc.xuntagged()

67

68 if !uidonly {

69 // Search with modseq search criteria.

70 tc.transactf("ok", "Search Modseq 0") // Zero is valid, matches all.

71 tc.xsearch()

72

73 tc.transactf("ok", "Search Modseq 1") // Converted to zero internally.

74 tc.xsearch()

75

76 tc.transactf("ok", "Search Modseq 12345")

77 tc.xsearch()

78

79 tc.transactf("ok", `Search Modseq "/Flags/\\Draft" All 12345`)

80 tc.xsearch()

81

82 tc.transactf("ok", `Search Or Modseq 12345 Modseq 54321`)

83 tc.xsearch()

84

85 // esearch

86 tc.transactf("ok", "Search Return (All) Modseq 123")

87 tc.xesearch(imapclient.UntaggedEsearch{})

88 }

89

90 // Now we add, delete, expunge, modify some message flags and check if the

91 // responses are correct. We check in both a condstore-enabled and one without that

92 // we get the correct notifications.

93

94 // First we add 3 messages as if they were added before we implemented CONDSTORE.

95 // Later on, we'll update the second, and delete the third, leaving the first

96 // unmodified. Those messages have modseq 0 in the database. We use append for

97 // convenience, then adjust the records in the database.

98 // We have a workaround below to prevent triggering the consistency checker.

99 tc.account.SetSkipMessageModSeqZeroCheck(true)

100 defer tc.account.SetSkipMessageModSeqZeroCheck(false)

101 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")

102 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")

103 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")

104 _, err := bstore.QueryDB[store.Message](ctxbg, tc.account.DB).UpdateFields(map[string]any{

105 "ModSeq": 0,

106 "CreateSeq": 0,

107 })

108 tcheck(t, err, "clearing modseq from messages")

109 err = tc.account.DB.Update(ctxbg, &store.SyncState{ID: 1, LastModSeq: 1})

110 tcheck(t, err, "resetting modseq state")

111

112 tc.client.Create("otherbox", nil)

113

114 // tc2 is a client without condstore, so no modseq responses.

115 tc2 := startNoSwitchboard(t, uidonly)

116 defer tc2.closeNoWait()

117 tc2.login("mjl@mox.example", password0)

118 tc2.client.Select("inbox")

119

120 // tc3 is a client with condstore, so with modseq responses.

121 tc3 := startNoSwitchboard(t, uidonly)

122 defer tc3.closeNoWait()

123 tc3.login("mjl@mox.example", password0)

124 tc3.client.Enable(capability)

125 tc3.client.Select("inbox")

126

127 var clientModseq int64 = 2 // We track the client-side modseq for inbox. Not a store.ModSeq.

128

129 // Add messages to: inbox, otherbox, inbox, inbox.

130 // We have these messages in order of modseq: 2+1 in inbox, 1 in otherbox, 2 in inbox.

131 // The original two in inbox appear to have modseq 1 (with 0 stored in the database).

132 // Creation of otherbox got modseq 2.

133 // The ones we insert below will start with modseq 3. So we'll have messages with modseq 1 and 3-6.

134 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")

135 tc.xuntagged(imapclient.UntaggedExists(4))

136 tc.xcode(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("4")})

137

138 tc.transactf("ok", "Append otherbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")

139 tc.xuntagged()

140 tc.xcode(imapclient.CodeAppendUID{UIDValidity: 3, UIDs: xparseUIDRange("1")})

141

142 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")

143 tc.xuntagged(imapclient.UntaggedExists(5))

144 tc.xcode(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("5")})

145

146 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")

147 tc.xuntagged(imapclient.UntaggedExists(6))

148 tc.xcode(imapclient.CodeAppendUID{UIDValidity: 1, UIDs: xparseUIDRange("6")})

149

150 tc2.transactf("ok", "Noop")

151 noflags := imapclient.FetchFlags(nil)

152 tc2.xuntagged(

153 imapclient.UntaggedExists(6),

154 tc2.untaggedFetch(4, 4, noflags),

155 tc2.untaggedFetch(5, 5, noflags),

156 tc2.untaggedFetch(6, 6, noflags),

157 )

158

159 tc3.transactf("ok", "Noop")

160 tc3.xuntagged(

161 imapclient.UntaggedExists(6),

162 tc3.untaggedFetch(4, 4, noflags, imapclient.FetchModSeq(clientModseq+1)),

163 tc3.untaggedFetch(5, 5, noflags, imapclient.FetchModSeq(clientModseq+3)),

164 tc3.untaggedFetch(6, 6, noflags, imapclient.FetchModSeq(clientModseq+4)),

165 )

166

167 if !uidonly {

168 mox.SetPedantic(true)

169 tc.transactf("bad", `Fetch 1 Flags (Changedsince 0)`) // 0 not allowed in syntax.

170 mox.SetPedantic(false)

171 }

172 tc.transactf("ok", "Uid fetch 1 (Flags) (Changedsince 0)")

173 tc.xuntagged(tc.untaggedFetch(1, 1, noflags, imapclient.FetchModSeq(1)))

174

175 // Check highestmodseq for mailboxes.

176 tc.transactf("ok", "Status inbox (highestmodseq)")

177 tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusHighestModSeq: clientModseq + 4}})

178

179 tc.transactf("ok", "Status otherbox (highestmodseq)")

180 tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "otherbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusHighestModSeq: clientModseq + 2}})

181

182 // Check highestmodseq when we select.

183 tc.transactf("ok", "Examine otherbox")

184 tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeHighestModSeq(clientModseq + 2), Text: "x"})

185

186 tc.transactf("ok", "Select inbox")

187 tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeHighestModSeq(clientModseq + 4), Text: "x"})

188

189 clientModseq += 4

190

191 if !uidonly {

192 // Check fetch modseq response and changedsince.

193 tc.transactf("ok", `Fetch 1 (Modseq)`)

194 tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchModSeq(1)))

195 }

196

197 // Without modseq attribute, even with condseq enabled, there is no modseq response.

198 // For QRESYNC, we must always send MODSEQ for UID FETCH commands, but not for FETCH commands. ../rfc/7162:1427

199 tc.transactf("ok", `Uid Fetch 1 Flags`)

200 if qresync {

201 tc.xuntagged(tc.untaggedFetch(1, 1, noflags, imapclient.FetchModSeq(1)))

202 } else {

203 tc.xuntagged(tc.untaggedFetch(1, 1, noflags))

204 }

205

206 if !uidonly {

207 tc.transactf("ok", `Fetch 1 Flags`)

208 tc.xuntagged(tc.untaggedFetch(1, 1, noflags))

209 }

210

211 if !uidonly {

212 // When CHANGEDSINCE is present, MODSEQ is automatically added to the response.

213 // ../rfc/7162:871

214 // ../rfc/7162:877

215 tc.transactf("ok", `Fetch 1 Flags (Changedsince 1)`)

216 tc.xuntagged()

217 tc.transactf("ok", `Fetch 1,4 Flags (Changedsince 1)`)

218 tc.xuntagged(tc.untaggedFetch(4, 4, noflags, imapclient.FetchModSeq(3)))

219 tc.transactf("ok", `Fetch 2 Flags (Changedsince 2)`)

220 tc.xuntagged()

221 }

222

223 // store and uid store.

224

225 if !uidonly {

226 // unchangedsince 0 never passes the check. ../rfc/7162:640

227 tc.transactf("ok", `Store 1 (Unchangedsince 0) +Flags ()`)

228 tc.xcode(imapclient.CodeModified(xparseNumSet("1")))

229 tc.xuntagged(tc.untaggedFetch(1, 1, noflags, imapclient.FetchModSeq(1)))

230 }

231

232 // Modseq is 2 for first condstore-aware-appended message, so also no match.

233 tc.transactf("ok", `Uid Store 4 (Unchangedsince 1) +Flags ()`)

234 tc.xcode(imapclient.CodeModified(xparseNumSet("4")))

235

236 if uidonly {

237 tc.transactf("ok", `Uid Store 1 (Unchangedsince 1) +Flags (label1)`)

238 } else {

239 // Modseq is 1 for original message.

240 tc.transactf("ok", `Store 1 (Unchangedsince 1) +Flags (label1)`)

241 }

242 tc.xcode(nil) // No MODIFIED.

243 clientModseq++

244 tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)))

245 tc2.transactf("ok", "Noop")

246 tc2.xuntagged(

247 tc2.untaggedFetch(1, 1, imapclient.FetchFlags{"label1"}),

248 )

249 tc3.transactf("ok", "Noop")

250 tc3.xuntagged(

251 tc3.untaggedFetch(1, 1, imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)),

252 )

253

254 // Modify same message twice. Check that second application doesn't fail due to

255 // modseq change made in the first application. ../rfc/7162:823

256 tc.transactf("ok", `Uid Store 1,1 (Unchangedsince %d) -Flags (label1)`, clientModseq)

257 clientModseq++

258 tc.xcode(nil) // No MODIFIED.

259 tc.xuntagged(

260 tc.untaggedFetch(1, 1, imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)),

261 )

262 // We do broadcast the changes twice. Not great, but doesn't hurt. This isn't common.

263 tc2.transactf("ok", "Noop")

264 tc2.xuntagged(

265 tc2.untaggedFetch(1, 1, imapclient.FetchFlags(nil)),

266 )

267 tc3.transactf("ok", "Noop")

268 tc3.xuntagged(

269 tc3.untaggedFetch(1, 1, imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)),

270 )

271

272 if !uidonly {

273 // Modify without actually changing flags, there will be no new modseq and no broadcast.

274 tc.transactf("ok", `Store 1 (Unchangedsince %d) -Flags (label1)`, clientModseq)

275 tc.xuntagged(tc.untaggedFetch(1, 1, imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)))

276 tc.xcode(nil) // No MODIFIED.

277 tc2.transactf("ok", "Noop")

278 tc2.xuntagged()

279 tc3.transactf("ok", "Noop")

280 tc3.xuntagged()

281

282 // search with modseq criteria and modseq in response

283 tc.transactf("ok", "Search Modseq %d", clientModseq)

284 tc.xsearchmodseq(clientModseq, 1)

285 }

286

287 tc.transactf("ok", "Uid Search Or Modseq %d Modseq %d", clientModseq, clientModseq)

288 tc.xsearchmodseq(clientModseq, 1)

289

290 if !uidonly {

291 // esearch

292 tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq %d", clientModseq)

293 tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: clientModseq})

294

295 tc.transactf("ok", "Search Return (Count) 1:* Modseq 0")

296 tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(6), ModSeq: clientModseq})

297

298 tc.transactf("ok", "Search Return (Min Max) 1:* Modseq 0")

299 tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 6, ModSeq: clientModseq})

300

301 tc.transactf("ok", "Search Return (Min) 1:* Modseq 0")

302 tc.xesearch(imapclient.UntaggedEsearch{Min: 1, ModSeq: clientModseq})

303

304 // expunge, we expunge the third and fourth messages. The third was originally with

305 // modseq 0, the fourth was added with condstore-aware append.

306 tc.transactf("ok", `Store 3:4 +Flags (\Deleted)`)

307 clientModseq++

308 } else {

309 tc.transactf("ok", `Uid Store 3,4 +Flags (\Deleted)`)

310 clientModseq++

311 }

312 tc2.transactf("ok", "Noop")

313 tc3.transactf("ok", "Noop")

314 tc.transactf("ok", "Expunge")

315 clientModseq++

316 if qresync || uidonly {

317 tc.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})

318 } else {

319 tc.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))

320 }

321 tc.xcode(imapclient.CodeHighestModSeq(clientModseq))

322 tc2.transactf("ok", "Noop")

323 if uidonly {

324 tc2.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})

325 } else {

326 tc2.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))

327 }

328 tc3.transactf("ok", "Noop")

329 if qresync || uidonly {

330 tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})

331 } else {

332 tc3.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))

333 }

334

335 // Again after expunge: status, select, conditional store/fetch/search

336 tc.transactf("ok", "Status inbox (Highestmodseq Messages Unseen Deleted)")

337 tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[imapclient.StatusAttr]int64{imapclient.StatusMessages: 4, imapclient.StatusUnseen: 4, imapclient.StatusDeleted: 0, imapclient.StatusHighestModSeq: clientModseq}})

338

339 tc.transactf("ok", "Close")

340 tc.transactf("ok", "Select inbox")

341 tc.xuntaggedOpt(false,

342 imapclient.UntaggedExists(4),

343 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeHighestModSeq(clientModseq), Text: "x"},

344 )

345

346 if !uidonly {

347 tc.transactf("ok", `Fetch 1:* (Modseq)`)

348 tc.xuntagged(

349 tc.untaggedFetch(1, 1, imapclient.FetchModSeq(8)),

350 tc.untaggedFetch(2, 2, imapclient.FetchModSeq(1)),

351 tc.untaggedFetch(3, 5, imapclient.FetchModSeq(5)),

352 tc.untaggedFetch(4, 6, imapclient.FetchModSeq(6)),

353 )

354 }

355 // Expunged messages, with higher modseq, should not show up.

356 tc.transactf("ok", "Uid Fetch 1:* (flags) (Changedsince 8)")

357 tc.xuntagged()

358

359 if !uidonly {

360 // search

361 tc.transactf("ok", "Search Modseq 8")

362 tc.xsearchmodseq(8, 1)

363 tc.transactf("ok", "Search Modseq 9")

364 tc.xsearch()

365

366 // esearch

367 tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 8")

368 tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: 8})

369 tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 9")

370 tc.xuntagged(imapclient.UntaggedEsearch{Tag: tc.client.LastTag()})

371 }

372

373 // store, cannot modify expunged messages.

374 tc.transactf("ok", `Uid Store 3,4 (Unchangedsince %d) +Flags (label2)`, clientModseq)

375 tc.xuntagged()

376 tc.xcode(nil) // Not MODIFIED.

377 tc.transactf("ok", `Uid Store 3,4 +Flags (label2)`)

378 tc.xuntagged()

379 tc.xcode(nil) // Not MODIFIED.

380

381 // Check all condstore-enabling commands (and their syntax), ../rfc/7162:368

382

383 // We start a new connection, do the thing that should enable condstore, then

384 // change flags of a message in another connection, do a noop in the new connection

385 // which should result in an untagged fetch that includes modseq, the indicator

386 // that condstore was indeed enabled. It's a bit complicated, but i don't think

387 // there is a clearly specified mechanism to find out which capabilities are

388 // enabled at any point.

389 var tagcount int

390 checkCondstoreEnabled := func(fn func(xtc *testconn)) {

391 t.Helper()

392

393 xtc := startNoSwitchboard(t, uidonly)

394 // We have modified modseq & createseq to 0 above for testing that case. Don't

395 // trigger the consistency checker.

396 defer xtc.closeNoWait()

397 xtc.login("mjl@mox.example", password0)

398 fn(xtc)

399 tagcount++

400 label := fmt.Sprintf("l%d", tagcount)

401 tc.transactf("ok", "Uid Store 6 Flags (%s)", label)

402 clientModseq++

403 xtc.transactf("ok", "Noop")

404 xtc.xuntagged(xtc.untaggedFetch(4, 6, imapclient.FetchFlags{label}, imapclient.FetchModSeq(clientModseq)))

405 }

406 // SELECT/EXAMINE with CONDSTORE parameter, ../rfc/7162:373

407 checkCondstoreEnabled(func(xtc *testconn) {

408 t.Helper()

409 xtc.transactf("ok", "Select inbox (Condstore)")

410 })

411 // STATUS with HIGHESTMODSEQ attribute, ../rfc/7162:375

412 checkCondstoreEnabled(func(xtc *testconn) {

413 t.Helper()

414 xtc.transactf("ok", "Status otherbox (Highestmodseq)")

415 xtc.transactf("ok", "Select inbox")

416 })

417 // FETCH with MODSEQ ../rfc/7162:377

418 checkCondstoreEnabled(func(xtc *testconn) {

419 t.Helper()

420 xtc.transactf("ok", "Select inbox")

421 xtc.transactf("ok", "Uid Fetch 6 (Modseq)")

422 })

423 // SEARCH with MODSEQ ../rfc/7162:377

424 checkCondstoreEnabled(func(xtc *testconn) {

425 t.Helper()

426 xtc.transactf("ok", "Select inbox")

427 xtc.transactf("ok", "Uid Search Uid 6 Modseq 1")

428 })

429 // FETCH with CHANGEDSINCE ../rfc/7162:380

430 checkCondstoreEnabled(func(xtc *testconn) {

431 t.Helper()

432 xtc.transactf("ok", "Select inbox")

433 xtc.transactf("ok", "Uid Fetch 6 (Flags) (Changedsince %d)", clientModseq)

434 })

435 // STORE with UNCHANGEDSINCE ../rfc/7162:382

436 checkCondstoreEnabled(func(xtc *testconn) {

437 t.Helper()

438 xtc.transactf("ok", "Select inbox")

439 xtc.transactf("ok", "Uid Store 6 (Unchangedsince 0) Flags ()")

440 })

441 // ENABLE CONDSTORE ../rfc/7162:384

442 checkCondstoreEnabled(func(xtc *testconn) {

443 t.Helper()

444 xtc.transactf("ok", "Enable Condstore")

445 xtc.transactf("ok", "Select inbox")

446 })

447 // ENABLE QRESYNC ../rfc/7162:1390

448 checkCondstoreEnabled(func(xtc *testconn) {

449 t.Helper()

450 xtc.transactf("ok", "Enable Qresync")

451 xtc.transactf("ok", "Select inbox")

452 })

453

454 if qresync {

455 tc.transactf("ok", "Uid Store 6 Flags ()")

456 clientModseq++

457

458 testQresync(t, tc, uidonly, clientModseq)

459 }

460

461 // Continue with some tests that further change the data.

462 // First we copy messages to a new mailbox, and check we get new modseq for those

463 // messages.

464 tc.transactf("ok", "Select otherbox")

465 tc2.transactf("ok", "Noop")

466 tc3.transactf("ok", "Noop")

467 tc.transactf("ok", "Uid Copy 1 inbox")

468 clientModseq++

469 tc2.transactf("ok", "Noop")

470 tc3.transactf("ok", "Noop")

471 tc2.xuntagged(

472 imapclient.UntaggedExists(5),

473 tc2.untaggedFetch(5, 7, noflags),

474 )

475 tc3.xuntagged(

476 imapclient.UntaggedExists(5),

477 tc3.untaggedFetch(5, 7, noflags, imapclient.FetchModSeq(clientModseq)),

478 )

479

480 // Then we move some messages, and check if we get expunged/vanished in original

481 // and untagged fetch with modseq in destination mailbox.

482 // tc2o is a client without condstore, so no modseq responses.

483 tc2o := startNoSwitchboard(t, uidonly)

484 defer tc2o.closeNoWait()

485 tc2o.login("mjl@mox.example", password0)

486 tc2o.client.Select("otherbox")

487

488 // tc3o is a client with condstore, so with modseq responses.

489 tc3o := startNoSwitchboard(t, uidonly)

490 defer tc3o.closeNoWait()

491 tc3o.login("mjl@mox.example", password0)

492 tc3o.client.Enable(capability)

493 tc3o.client.Select("otherbox")

494

495 tc.transactf("ok", "Select inbox")

496 tc.transactf("ok", "Uid Move 2:4 otherbox") // Only UID 2, because UID 3 and 4 have already been expunged.

497 clientModseq++

498 if qresync {

499 tc.xuntaggedOpt(false, imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})

500 tc.xcode(imapclient.CodeHighestModSeq(clientModseq))

501 } else if uidonly {

502 tc.xuntaggedOpt(false, imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})

503 tc.xcode(nil)

504 } else {

505 tc.xuntaggedOpt(false, imapclient.UntaggedExpunge(2))

506 tc.xcode(nil)

507 }

508 tc2.transactf("ok", "Noop")

509 if uidonly {

510 tc2.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})

511 } else {

512 tc2.xuntagged(imapclient.UntaggedExpunge(2))

513 }

514 tc3.transactf("ok", "Noop")

515 if qresync || uidonly {

516 tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})

517 } else {

518 tc3.xuntagged(imapclient.UntaggedExpunge(2))

519 }

520 tc2o.transactf("ok", "Noop")

521 tc2o.xuntagged(

522 imapclient.UntaggedExists(2),

523 tc2o.untaggedFetch(2, 2, noflags),

524 )

525 tc3o.transactf("ok", "Noop")

526 tc3o.xuntagged(

527 imapclient.UntaggedExists(2),

528 tc2o.untaggedFetch(2, 2, noflags, imapclient.FetchModSeq(clientModseq)),

529 )

530

531 tc2o.closeNoWait()

532 tc2o = nil

533 tc3o.closeNoWait()

534 tc3o = nil

535

536 // Then we rename inbox, which is special because it moves messages away instead of

537 // actually moving the mailbox. The mailbox stays and is cleared, so we check if we

538 // get expunged/vanished messages.

539 tc.transactf("ok", "Rename inbox oldbox")

540 // todo spec: server doesn't respond with untagged responses, find rfc reference that says this is ok.

541 tc2.transactf("ok", "Noop")

542 if uidonly {

543 tc2.xuntagged(

544 imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},

545 imapclient.UntaggedVanished{UIDs: xparseNumSet("1,5:7")},

546 )

547 } else {

548 tc2.xuntagged(

549 imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},

550 imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),

551 )

552 }

553 tc3.transactf("ok", "Noop")

554 if qresync || uidonly {

555 tc3.xuntagged(

556 imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},

557 imapclient.UntaggedVanished{UIDs: xparseNumSet("1,5:7")},

558 )

559 } else {

560 tc3.xuntagged(

561 imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},

562 imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),

563 )

564 }

565

566 // Then we delete otherbox (we cannot delete inbox). We don't keep any history for removed mailboxes, so not actually a special case.

567 tc.transactf("ok", "Delete otherbox")

568}

569

570func testQresync(t *testing.T, tc *testconn, uidonly bool, clientModseq int64) {

571 // Vanished on non-uid fetch is not allowed. ../rfc/7162:1693

572 tc.transactf("bad", "fetch 1:* (Flags) (Changedsince 1 Vanished)")

573

574 // Vanished without changedsince is not allowed. ../rfc/7162:1701

575 tc.transactf("bad", "Uid Fetch 1:* (Flags) (Vanished)")

576

577 // Vanished not allowed without first enabling qresync. ../rfc/7162:1697

578 xtc := startNoSwitchboard(t, uidonly)

579 xtc.login("mjl@mox.example", password0)

580 xtc.transactf("ok", "Select inbox (Condstore)")

581 xtc.transactf("bad", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")

582 xtc.closeNoWait()

583 xtc = nil

584

585 // Check that we get proper vanished responses.

586 tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")

587 noflags := imapclient.FetchFlags(nil)

588 tc.xuntagged(

589 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},

590 tc.untaggedFetch(3, 5, noflags, imapclient.FetchModSeq(5)),

591 tc.untaggedFetch(1, 1, noflags, imapclient.FetchModSeq(8)),

592 tc.untaggedFetch(4, 6, noflags, imapclient.FetchModSeq(clientModseq)),

593 )

594

595 // select/examine with qresync parameters, including the various optional fields.

596 tc.transactf("ok", "Close")

597

598 // Must enable qresync explicitly before using. ../rfc/7162:1446

599 xtc = startNoSwitchboard(t, uidonly)

600 xtc.login("mjl@mox.example", password0)

601 xtc.transactf("bad", "Select inbox (Qresync 1 0)")

602 // Prevent triggering the consistency checker, we still have modseq/createseq at 0.

603 xtc.closeNoWait()

604 xtc = nil

605

606 tc.transactf("bad", "Select inbox (Qresync (0 1))") // Both args must be > 0.

607 tc.transactf("bad", "Select inbox (Qresync (1 0))") // Both args must be > 0.

608 tc.transactf("bad", "Select inbox (Qresync)") // Two args are minimum.

609 tc.transactf("bad", "Select inbox (Qresync (1))") // Two args are minimum.

610 tc.transactf("bad", "Select inbox (Qresync (1 1 1:*))") // Known UIDs, * not allowed.

611 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:6)))") // Known seqset cannot have *.

612 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:6 1:*)))") // Known uidset cannot have *.

613 tc.transactf("bad", "Select inbox (Qresync (1 1) qresync (1 1))") // Duplicate qresync.

614

615 flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent l1 l2 l3 l4 l5 l6 l7 l8 label1`, " ")

616 permflags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*`, " ")

617 uflags := imapclient.UntaggedFlags(flags)

618 upermflags := imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodePermanentFlags(permflags), Text: "x"}

619

620 baseUntagged := []imapclient.Untagged{

621 uflags,

622 upermflags,

623 imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"},

624 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeUIDNext(7), Text: "x"},

625 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeUIDValidity(1), Text: "x"},

626 imapclient.UntaggedRecent(0),

627 imapclient.UntaggedExists(4),

628 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeHighestModSeq(clientModseq), Text: "x"},

629 }

630 if !uidonly {

631 baseUntagged = append(baseUntagged,

632 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeUnseen(1), Text: "x"},

633 )

634 }

635

636 makeUntagged := func(l ...imapclient.Untagged) []imapclient.Untagged {

637 return slices.Concat(baseUntagged, l)

638 }

639

640 // uidvalidity 1, highest known modseq 1, sends full current state.

641 tc.transactf("ok", "Select inbox (Qresync (1 1))")

642 tc.xuntagged(

643 makeUntagged(

644 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},

645 tc.untaggedFetch(3, 5, noflags, imapclient.FetchModSeq(5)),

646 tc.untaggedFetch(1, 1, noflags, imapclient.FetchModSeq(8)),

647 tc.untaggedFetch(4, 6, noflags, imapclient.FetchModSeq(clientModseq)),

648 )...,

649 )

650

651 // Uidvalidity mismatch, server will not send any changes, so it's just a regular open.

652 tc.transactf("ok", "Close")

653 tc.transactf("ok", "Select inbox (Qresync (2 1))")

654 tc.xuntagged(baseUntagged...)

655

656 // We can tell which UIDs we know. First, send broader range then exist, should work.

657 tc.transactf("ok", "Close")

658 tc.transactf("ok", "Select inbox (Qresync (1 1 1:7))")

659 tc.xuntagged(

660 makeUntagged(

661 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},

662 tc.untaggedFetch(3, 5, noflags, imapclient.FetchModSeq(5)),

663 tc.untaggedFetch(1, 1, noflags, imapclient.FetchModSeq(8)),

664 tc.untaggedFetch(4, 6, noflags, imapclient.FetchModSeq(clientModseq)),

665 )...,

666 )

667

668 // Now send just the ones that exist. We won't get the vanished messages.

669 tc.transactf("ok", "Close")

670 tc.transactf("ok", "Select inbox (Qresync (1 1 1,2,5:6))")

671 tc.xuntagged(

672 makeUntagged(

673 tc.untaggedFetch(3, 5, noflags, imapclient.FetchModSeq(5)),

674 tc.untaggedFetch(1, 1, noflags, imapclient.FetchModSeq(8)),

675 tc.untaggedFetch(4, 6, noflags, imapclient.FetchModSeq(clientModseq)),

676 )...,

677 )

678

679 // We'll only get updates for UIDs we specify.

680 tc.transactf("ok", "Close")

681 tc.transactf("ok", "Select inbox (Qresync (1 1 5))")

682 tc.xuntagged(

683 makeUntagged(

684 tc.untaggedFetch(3, 5, noflags, imapclient.FetchModSeq(5)),

685 )...,

686 )

687

688 // We'll only get updates for UIDs we specify. ../rfc/7162:1523

689 tc.transactf("ok", "Close")

690 tc.transactf("ok", "Select inbox (Qresync (1 1 3))")

691 tc.xuntagged(

692 makeUntagged(

693 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3")},

694 )...,

695 )

696

697 // If we specify the latest modseq, we'll get no changes.

698 tc.transactf("ok", "Close")

699 tc.transactf("ok", "Select inbox (Qresync (1 %d))", clientModseq)

700 tc.xuntagged(baseUntagged...)

701

702 // We can provide our own seqs & uids, and have server determine which uids we

703 // know. But the seqs & uids must be of equal length. First try with a few combinations

704 // that aren't valid. ../rfc/7162:1579

705 tc.transactf("ok", "Close")

706 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1 1,2)))") // Not same length.

707 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1,2 1)))") // Not same length.

708 if !uidonly {

709 tc.transactf("no", "Select inbox (Qresync (1 1 1:6 (1,2 1,1)))") // Not ascending.

710 }

711 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:4)))") // Star not allowed.

712

713 if uidonly {

714 return

715 }

716

717 // With valid parameters, based on what a client would know at this stage.

718 tc.transactf("ok", "Select inbox (Qresync (1 1 1:6 (1,3,6 1,3,6)))")

719 tc.xuntagged(

720 makeUntagged(

721 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},

722 tc.untaggedFetch(3, 5, noflags, imapclient.FetchModSeq(5)),

723 tc.untaggedFetch(1, 1, noflags, imapclient.FetchModSeq(8)),

724 tc.untaggedFetch(4, 6, noflags, imapclient.FetchModSeq(clientModseq)),

725 )...,

726 )

727

728 // The 3rd parameter is optional, try without.

729 tc.transactf("ok", "Close")

730 tc.transactf("ok", "Select inbox (Qresync (1 5 (1,3,6 1,3,6)))")

731 tc.xuntagged(

732 makeUntagged(

733 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},

734 tc.untaggedFetch(1, 1, noflags, imapclient.FetchModSeq(8)),

735 tc.untaggedFetch(4, 6, noflags, imapclient.FetchModSeq(clientModseq)),

736 )...,

737 )

738

739 tc.transactf("ok", "Close")

740 tc.transactf("ok", "Select inbox (Qresync (1 9 (1,3,6 1,3,6)))")

741 tc.xuntagged(

742 makeUntagged(

743 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},

744 tc.untaggedFetch(4, 6, noflags, imapclient.FetchModSeq(clientModseq)),

745 )...,

746 )

747

748 // Client will claim a highestmodseq but then include uids that have been removed

749 // since that time. Server detects this, sends full vanished history and continues

750 // working with modseq changed to 1 before the expunged uid.

751 tc.transactf("ok", "Close")

752 tc.transactf("ok", "Select inbox (Qresync (1 10 (1,3,6 1,3,6)))")

753 tc.xuntagged(

754 makeUntagged(

755 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeWord("ALERT"), Text: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full synchronization recommended."},

756 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},

757 tc.untaggedFetch(4, 6, noflags, imapclient.FetchModSeq(clientModseq)),

758 )...,

759 )

760

761 // Client will claim a highestmodseq but then include uids that have been removed

762 // since that time. Server detects this, sends full vanished history and continues

763 // working with modseq changed to 1 before the expunged uid.

764 tc.transactf("ok", "Close")

765 tc.transactf("ok", "Select inbox (Qresync (1 18 (1,3,6 1,3,6)))")

766 tc.xuntagged(

767 makeUntagged(

768 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeWord("ALERT"), Text: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full synchronization recommended."},

769 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},

770 tc.untaggedFetch(4, 6, noflags, imapclient.FetchModSeq(clientModseq)),

771 )...,

772 )

773}

774

775func TestQresyncHistory(t *testing.T) {

776 testQresyncHistory(t, false)

777}

778

779func TestQresyncHistoryUIDOnly(t *testing.T) {

780 testQresyncHistory(t, true)

781}

782

783func testQresyncHistory(t *testing.T, uidonly bool) {

784 defer mockUIDValidity()()

785 tc := start(t, uidonly)

786 defer tc.close()

787

788 tc.login("mjl@mox.example", password0)

789 tc.client.Enable(imapclient.CapQresync)

790 tc.transactf("ok", "Append inbox {1+}\r\nx")

791 tc.transactf("ok", "Append inbox {1+}\r\nx") // modseq 6

792 tc.transactf("ok", "Append inbox {1+}\r\nx")

793 tc.transactf("ok", "Select inbox")

794 tc.client.UIDStoreFlagsAdd("1,3", true, `\Deleted`) // modseq 8

795 tc.client.Expunge() // modseq 9

796 tc.client.UIDStoreFlagsAdd("2", true, `\Seen`) // modseq 10

797 // We have UID 2, no more UID 1 and 3.

798

799 flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent`, " ")

800 permflags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*`, " ")

801 uflags := imapclient.UntaggedFlags(flags)

802 upermflags := imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodePermanentFlags(permflags), Text: "x"}

803 baseUntagged := []imapclient.Untagged{

804 uflags,

805 upermflags,

806 imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"},

807 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeUIDNext(4), Text: "x"},

808 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeUIDValidity(1), Text: "x"},

809 imapclient.UntaggedRecent(0),

810 imapclient.UntaggedExists(1),

811 imapclient.UntaggedResult{Status: imapclient.OK, Code: imapclient.CodeHighestModSeq(10), Text: "x"},

812 }

813

814 makeUntagged := func(l ...imapclient.Untagged) []imapclient.Untagged {

815 return slices.Concat(baseUntagged, l)

816 }

817

818 tc.transactf("ok", "Close")

819 tc.transactf("ok", "Select inbox (Qresync (1 1))")

820 tc.xuntagged(

821 makeUntagged(

822 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("1,3")},

823 tc.untaggedFetch(1, 2, imapclient.FetchFlags{`\Seen`}, imapclient.FetchModSeq(10)),

824 )...,

825 )

826

827 err := tc.account.DB.Write(ctxbg, func(tx *bstore.Tx) error {

828 syncState := store.SyncState{ID: 1}

829 err := tx.Get(&syncState)

830 tcheck(t, err, "get syncstate")

831

832 syncState.HighestDeletedModSeq = 9

833 err = tx.Update(&syncState)

834 tcheck(t, err, "update syncstate")

835

836 q := bstore.QueryTx[store.Message](tx)

837 q.FilterNonzero(store.Message{Expunged: true})

838 q.FilterLessEqual("ModSeq", syncState.HighestDeletedModSeq)

839 n, err := q.Delete()

840 tcheck(t, err, "delete history")

841 if n != 2 {

842 t.Fatalf("removed %d message history records, expected 2", n)

843 }

844 return nil

845 })

846 tcheck(t, err, "db write")

847

848 // We should still get VANISHED EARLIER for 1,3, even though we don't have history for it.

849 tc.transactf("ok", "Close")

850 tc.transactf("ok", "Select inbox (Qresync (1 1))")

851 tc.xuntagged(

852 makeUntagged(

853 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("1,3")},

854 tc.untaggedFetch(1, 2, imapclient.FetchFlags{`\Seen`}, imapclient.FetchModSeq(10)),

855 )...,

856 )

857

858 // Similar with explicit UIDs.

859 tc.transactf("ok", "Close")

860 tc.transactf("ok", "Select inbox (Qresync (1 1 1:3))")

861 tc.xuntagged(

862 makeUntagged(

863 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("1,3")},

864 tc.untaggedFetch(1, 2, imapclient.FetchFlags{`\Seen`}, imapclient.FetchModSeq(10)),

865 )...,

866 )

867

868 // Fetch with changedsince also returns VANISHED EARLIER when we don't have history anymore.

869 tc.transactf("ok", "uid fetch 1:3 flags (Changedsince 10)")

870 tc.xuntagged() // We still have history, nothing changed.

871

872 tc.transactf("ok", "uid fetch 1:3 flags (Changedsince 9)")

873 tc.xuntagged(tc.untaggedFetch(1, 2, imapclient.FetchFlags{`\Seen`}, imapclient.FetchModSeq(10)))

874

875 // Missing history, but no vanished requested.

876 tc.transactf("ok", "uid fetch 1:4 flags (Changedsince 1)")

877 tc.xuntagged(

878 tc.untaggedFetch(1, 2, imapclient.FetchFlags{`\Seen`}, imapclient.FetchModSeq(10)),

879 )

880

881 // Same, but with vanished requested.

882 tc.transactf("ok", "uid fetch 1:3 flags (Vanished Changedsince 10)")

883 tc.xuntagged() // We still have history, nothing changed.

884

885 tc.transactf("ok", "uid fetch 1:3 flags (Vanished Changedsince 9)")

886 tc.xuntagged(tc.untaggedFetch(1, 2, imapclient.FetchFlags{`\Seen`}, imapclient.FetchModSeq(10)))

887

888 // We return vanished for 1,3. Not for 4, since that is uidnext.

889 tc.transactf("ok", "uid fetch 1:4 flags (Vanished Changedsince 1)")

890 tc.xuntagged(

891 tc.untaggedFetch(1, 2, imapclient.FetchFlags{`\Seen`}, imapclient.FetchModSeq(10)),

892 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("1,3")},

893 )

894}

895