1package store

2

3import (

4 "context"

5 "errors"

6 "io/fs"

7 "os"

8 "path/filepath"

9 "reflect"

10 "regexp"

11 "strings"

12 "testing"

13 "time"

14

15 "github.com/mjl-/bstore"

16 "github.com/mjl-/sconf"

17

18 "github.com/mjl-/mox/config"

19 "github.com/mjl-/mox/message"

20 "github.com/mjl-/mox/mlog"

21 "github.com/mjl-/mox/mox-"

22)

23

24var ctxbg = context.Background()

25var pkglog = mlog.New("store", nil)

26

27func tcheck(t *testing.T, err error, msg string) {

28 t.Helper()

29 if err != nil {

30 t.Fatalf("%s: %s", msg, err)

31 }

32}

33

34func tcompare(t *testing.T, got, expect any) {

35 t.Helper()

36 if !reflect.DeepEqual(got, expect) {

37 t.Fatalf("got:\n%#v\nexpected:\n%#v", got, expect)

38 }

39}

40

41func TestJunkFlagsForMailboxMove(t *testing.T) {

42 conf := config.Account{

43 Introbox: "Introbox",

44 AutomaticJunkFlags: config.AutomaticJunkFlags{

45 Enabled: true,

46 },

47 }

48 conf.JunkMailbox = regexp.MustCompile("^junk$")

49 conf.NotJunkMailbox = regexp.MustCompile("^(inbox|archive)$")

50

51 introbox := Mailbox{ID: 1, Name: "Introbox"}

52 inbox := Mailbox{ID: 2, Name: "Inbox"}

53 junkmb := Mailbox{ID: 3, Name: "Junk"}

54 archive := Mailbox{ID: 4, Name: "Archive"}

55 other := Mailbox{ID: 5, Name: "Other"}

56

57 m := Message{MailboxOrigID: introbox.ID, MailboxDestinedID: inbox.ID}

58 m.JunkFlagsForMailboxMove(introbox, inbox, conf)

59 if m.MailboxOrigID != inbox.ID || m.MailboxDestinedID != 0 || m.Junk || !m.Notjunk {

60 t.Fatalf("positive introbox move not recorded: %#v", m)

61 }

62

63 m = Message{MailboxOrigID: introbox.ID, MailboxDestinedID: inbox.ID}

64 m.JunkFlagsForMailboxMove(introbox, junkmb, conf)

65 if m.MailboxOrigID != inbox.ID || m.MailboxDestinedID != 0 || !m.Junk || m.Notjunk {

66 t.Fatalf("negative introbox move not recorded: %#v", m)

67 }

68

69 m = Message{MailboxOrigID: introbox.ID, MailboxDestinedID: inbox.ID}

70 m.JunkFlagsForMailboxMove(introbox, archive, conf)

71 if m.MailboxOrigID != inbox.ID || m.MailboxDestinedID != 0 || m.Junk || !m.Notjunk {

72 t.Fatalf("positive introbox move not recorded: %#v", m)

73 }

74

75 m = Message{MailboxOrigID: introbox.ID, MailboxDestinedID: inbox.ID}

76 m.JunkFlagsForMailboxMove(introbox, other, conf)

77 if m.MailboxOrigID != introbox.ID || m.MailboxDestinedID != inbox.ID || m.Junk || m.Notjunk {

78 t.Fatalf("unrelated introbox move changed reputation destination: %#v", m)

79 }

80}

81

82func TestMailbox(t *testing.T) {

83 log := mlog.New("store", nil)

84 os.RemoveAll("../testdata/store/data")

85 mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")

86 mox.MustLoadConfig(true, false)

87 err := Init(ctxbg)

88 tcheck(t, err, "init")

89 defer func() {

90 err := Close()

91 tcheck(t, err, "close")

92 }()

93 defer Switchboard()()

94 acc, err := OpenAccount(log, "mjl", false)

95 tcheck(t, err, "open account")

96 defer func() {

97 err = acc.Close()

98 tcheck(t, err, "closing account")

99 acc.WaitClosed()

100 }()

101

102 msgFile, err := CreateMessageTemp(log, "account-test")

103 tcheck(t, err, "create temp message file")

104 defer CloseRemoveTempFile(log, msgFile, "temp message file")

105 msgWriter := message.NewWriter(msgFile)

106 _, err = msgWriter.Write([]byte(" message"))

107 tcheck(t, err, "write message")

108

109 msgPrefix := []byte("From: <mjl@mox.example\r\nTo: <mjl@mox.example>\r\nCc: <mjl@mox.example>Subject: test\r\nMessage-Id: <m01@mox.example>\r\n\r\n")

110 msgPrefixCatchall := []byte("Subject: catchall\r\n\r\n")

111 m := Message{

112 Received: time.Now(),

113 Size: int64(len(msgPrefix)) + msgWriter.Size,

114 MsgPrefix: msgPrefix,

115 }

116 msent := m

117 m.ThreadMuted = true

118 m.ThreadCollapsed = true

119 var mbsent Mailbox

120 mreject := m

121 mconsumed := Message{

122 Received: m.Received,

123 Size: int64(len(msgPrefixCatchall)) + msgWriter.Size,

124 MsgPrefix: msgPrefixCatchall,

125 }

126 acc.WithWLock(func() {

127 conf, _ := acc.Conf()

128 err := acc.DeliverDestination(log, conf.Destinations["mjl"], &m, msgFile)

129 tcheck(t, err, "deliver without consume")

130

131 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

132 var err error

133 mbsent, err = bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Sent"}).Get()

134 tcheck(t, err, "sent mailbox")

135 msent.MailboxID = mbsent.ID

136 msent.MailboxOrigID = mbsent.ID

137 err = acc.MessageAdd(log, tx, &mbsent, &msent, msgFile, AddOpts{SkipSourceFileSync: true, SkipDirSync: true})

138 tcheck(t, err, "deliver message")

139 if !msent.ThreadMuted || !msent.ThreadCollapsed {

140 t.Fatalf("thread muted & collapsed should have been copied from parent (duplicate message-id) m")

141 }

142 err = tx.Update(&mbsent)

143 tcheck(t, err, "update mbsent")

144

145 modseq, err := acc.NextModSeq(tx)

146 tcheck(t, err, "get next modseq")

147 mbrejects := Mailbox{Name: "Rejects", UIDValidity: 1, UIDNext: 1, ModSeq: modseq, CreateSeq: modseq, HaveCounts: true}

148 err = tx.Insert(&mbrejects)

149 tcheck(t, err, "insert rejects mailbox")

150 mreject.MailboxID = mbrejects.ID

151 mreject.MailboxOrigID = mbrejects.ID

152 err = acc.MessageAdd(log, tx, &mbrejects, &mreject, msgFile, AddOpts{SkipSourceFileSync: true, SkipDirSync: true})

153 tcheck(t, err, "deliver message")

154 err = tx.Update(&mbrejects)

155 tcheck(t, err, "update mbrejects")

156

157 return nil

158 })

159 tcheck(t, err, "deliver as sent and rejects")

160

161 err = acc.DeliverDestination(log, conf.Destinations["mjl"], &mconsumed, msgFile)

162 tcheck(t, err, "deliver with consume")

163

164 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

165 m.Junk = true

166 l := []Message{m}

167 err = acc.RetrainMessages(ctxbg, log, tx, l)

168 tcheck(t, err, "train as junk")

169 m = l[0]

170 return nil

171 })

172 tcheck(t, err, "train messages")

173 })

174

175 m.Junk = false

176 m.Notjunk = true

177 jf, _, err := acc.OpenJunkFilter(ctxbg, log)

178 tcheck(t, err, "open junk filter")

179 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

180 return acc.RetrainMessage(ctxbg, log, tx, jf, &m)

181 })

182 tcheck(t, err, "retraining as non-junk")

183 err = jf.Close()

184 tcheck(t, err, "close junk filter")

185

186 m.Notjunk = false

187 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

188 return acc.RetrainMessages(ctxbg, log, tx, []Message{m})

189 })

190 tcheck(t, err, "untraining non-junk")

191

192 err = acc.SetPassword(log, "testtest")

193 tcheck(t, err, "set password")

194

195 key0, err := acc.Subjectpass("test@localhost")

196 tcheck(t, err, "subjectpass")

197 key1, err := acc.Subjectpass("test@localhost")

198 tcheck(t, err, "subjectpass")

199 if key0 != key1 {

200 t.Fatalf("different keys for same address")

201 }

202 key2, err := acc.Subjectpass("test2@localhost")

203 tcheck(t, err, "subjectpass")

204 if key2 == key0 {

205 t.Fatalf("same key for different address")

206 }

207

208 var modseq ModSeq

209 acc.WithWLock(func() {

210 var changes []Change

211

212 err := acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

213 _, _, err := acc.MailboxEnsure(tx, "Testbox", true, SpecialUse{}, &modseq)

214 return err

215 })

216 tcheck(t, err, "ensure mailbox exists")

217 err = acc.DB.Read(ctxbg, func(tx *bstore.Tx) error {

218 _, _, err := acc.MailboxEnsure(tx, "Testbox", true, SpecialUse{}, &modseq)

219 return err

220 })

221 tcheck(t, err, "ensure mailbox exists")

222

223 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

224 _, _, err := acc.MailboxEnsure(tx, "Testbox2", false, SpecialUse{}, &modseq)

225 tcheck(t, err, "create mailbox")

226

227 exists, err := acc.MailboxExists(tx, "Testbox2")

228 tcheck(t, err, "checking that mailbox exists")

229 if !exists {

230 t.Fatalf("mailbox does not exist")

231 }

232

233 exists, err = acc.MailboxExists(tx, "Testbox3")

234 tcheck(t, err, "checking that mailbox does not exist")

235 if exists {

236 t.Fatalf("mailbox does exist")

237 }

238

239 xmb, err := acc.MailboxFind(tx, "Testbox3")

240 tcheck(t, err, "finding non-existing mailbox")

241 if xmb != nil {

242 t.Fatalf("did find Testbox3: %v", xmb)

243 }

244 xmb, err = acc.MailboxFind(tx, "Testbox2")

245 tcheck(t, err, "finding existing mailbox")

246 if xmb == nil {

247 t.Fatalf("did not find Testbox2")

248 }

249

250 nchanges, err := acc.SubscriptionEnsure(tx, "Testbox2")

251 tcheck(t, err, "ensuring new subscription")

252 if len(nchanges) == 0 {

253 t.Fatalf("new subscription did not result in changes")

254 }

255 changes = append(changes, nchanges...)

256 nchanges, err = acc.SubscriptionEnsure(tx, "Testbox2")

257 tcheck(t, err, "ensuring already present subscription")

258 if len(nchanges) != 0 {

259 t.Fatalf("already present subscription resulted in changes")

260 }

261

262 // todo: check that messages are removed.

263 mbRej, err := bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Rejects"}).Get()

264 tcheck(t, err, "get rejects mailbox")

265 nchanges, hasSpace, err := acc.TidyRejectsMailbox(log, tx, &mbRej)

266 tcheck(t, err, "tidy rejects mailbox")

267 changes = append(changes, nchanges...)

268 if !hasSpace {

269 t.Fatalf("no space for more rejects")

270 }

271

272 return nil

273 })

274 tcheck(t, err, "write tx")

275

276 BroadcastChanges(acc, changes)

277

278 acc.RejectsRemove(log, "Rejects", "m01@mox.example")

279 })

280

281 // Run the auth tests twice for possible cache effects.

282 for range 2 {

283 _, _, err := OpenEmailAuth(log, "mjl@mox.example", "bogus", false)

284 if err != ErrUnknownCredentials {

285 t.Fatalf("got %v, expected ErrUnknownCredentials", err)

286 }

287 }

288

289 for range 2 {

290 acc2, _, err := OpenEmailAuth(log, "mjl@mox.example", "testtest", false)

291 tcheck(t, err, "open for email with auth")

292 err = acc2.Close()

293 tcheck(t, err, "close account")

294 }

295

296 acc2, _, err := OpenEmailAuth(log, "other@mox.example", "testtest", false)

297 tcheck(t, err, "open for email with auth")

298 err = acc2.Close()

299 tcheck(t, err, "close account")

300

301 _, _, err = OpenEmailAuth(log, "bogus@mox.example", "testtest", false)

302 if err != ErrUnknownCredentials {

303 t.Fatalf("got %v, expected ErrUnknownCredentials", err)

304 }

305

306 _, _, err = OpenEmailAuth(log, "mjl@test.example", "testtest", false)

307 if err != ErrUnknownCredentials {

308 t.Fatalf("got %v, expected ErrUnknownCredentials", err)

309 }

310}

311

312func TestMessageRuleset(t *testing.T) {

313 log := mlog.New("store", nil)

314 f, err := CreateMessageTemp(log, "msgruleset")

315 tcheck(t, err, "creating temp msg file")

316 defer CloseRemoveTempFile(log, f, "temp message file")

317

318 msgBuf := []byte(strings.ReplaceAll(`List-ID: <test.mox.example>

319

320test

321`, "\n", "\r\n"))

322

323 const destConf = `

324Rulesets:

325 -

326 HeadersRegexp:

327 list-id: <test\.mox\.example>

328 Mailbox: test

329`

330 var dest config.Destination

331 err = sconf.Parse(strings.NewReader(destConf), &dest)

332 tcheck(t, err, "parse config")

333 // todo: should use regular config initialization functions for this.

334 var hdrs [][2]*regexp.Regexp

335 for k, v := range dest.Rulesets[0].HeadersRegexp {

336 rk, err := regexp.Compile(k)

337 tcheck(t, err, "compile key")

338 rv, err := regexp.Compile(v)

339 tcheck(t, err, "compile value")

340 hdrs = append(hdrs, [...]*regexp.Regexp{rk, rv})

341 }

342 dest.Rulesets[0].HeadersRegexpCompiled = hdrs

343

344 c := MessageRuleset(log, dest, &Message{}, msgBuf, f)

345 if c == nil {

346 t.Fatalf("expected ruleset match")

347 }

348

349 msg2Buf := []byte(strings.ReplaceAll(`From: <mjl@mox.example>

350

351test

352`, "\n", "\r\n"))

353 c = MessageRuleset(log, dest, &Message{}, msg2Buf, f)

354 if c != nil {

355 t.Fatalf("expected no ruleset match")

356 }

357

358 // todo: test the SMTPMailFrom and VerifiedDomains rule.

359}

360

361// Check that opening an account forwards the Message.ID used for new additions if

362// message files already exist in the file system.

363func TestNextMessageID(t *testing.T) {

364 log := mlog.New("store", nil)

365 os.RemoveAll("../testdata/store/data")

366 mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")

367 mox.MustLoadConfig(true, false)

368 err := Init(ctxbg)

369 tcheck(t, err, "init")

370 defer func() {

371 err := Close()

372 tcheck(t, err, "close")

373 }()

374 defer Switchboard()()

375

376 // Ensure account exists.

377 acc, err := OpenAccount(log, "mjl", false)

378 tcheck(t, err, "open account")

379 err = acc.Close()

380 tcheck(t, err, "closing account")

381 acc.WaitClosed()

382 acc = nil

383

384 // Create file on disk to occupy the first Message.ID that would otherwise be used for deliveries..

385 msgData := []byte("a: b\r\n\r\ntest\r\n")

386 msgDir := filepath.FromSlash("../testdata/store/data/accounts/mjl/msg")

387 os.MkdirAll(filepath.Join(msgDir, "a"), 0700)

388 msgPath := filepath.Join(msgDir, "a", "1")

389 err = os.WriteFile(msgPath, msgData, 0700)

390 tcheck(t, err, "write message file")

391

392 msgPathBogus := filepath.Join(msgDir, "a", "bogus")

393 err = os.WriteFile(msgPathBogus, []byte("test"), 0700)

394 tcheck(t, err, "create message file")

395 msgPathBadID := filepath.Join(msgDir, "a", "10000") // Out of range.

396 err = os.WriteFile(msgPathBadID, []byte("test"), 0700)

397 tcheck(t, err, "create message file")

398

399 // Open account. This should increase the next message ID.

400 acc, err = OpenAccount(log, "mjl", false)

401 tcheck(t, err, "open account")

402

403 // Deliver a message. It should get ID 2.

404 mf, err := CreateMessageTemp(log, "account-test")

405 tcheck(t, err, "creating temp message file")

406 _, err = mf.Write(msgData)

407 tcheck(t, err, "write file")

408 defer CloseRemoveTempFile(log, mf, "temp message file")

409 m := Message{

410 Size: int64(len(msgData)),

411 }

412 err = acc.DeliverMailbox(log, "Inbox", "", &m, mf)

413 tcheck(t, err, "deliver mailbox")

414 if m.ID != 2 {

415 t.Fatalf("got message id %d, expected 2", m.ID)

416 }

417

418 // Ensure account consistency check won't complain.

419 err = os.Remove(msgPath)

420 tcheck(t, err, "removing message path")

421 err = os.Remove(msgPathBogus)

422 tcheck(t, err, "removing message path")

423 err = os.Remove(msgPathBadID)

424 tcheck(t, err, "removing message path")

425

426 err = acc.Close()

427 tcheck(t, err, "closing account")

428 acc.WaitClosed()

429

430 // Try again, but also create next message directory, but no file.

431 os.MkdirAll(filepath.Join(msgDir, "b"), 0700)

432 os.MkdirAll(filepath.Join(msgDir, "d"), 0700) // Not used.

433

434 // Open account again, increasing next message ID.

435 acc, err = OpenAccount(log, "mjl", false)

436 tcheck(t, err, "open account")

437

438 // Deliver a message. It should get ID $msgFilesPerDir+1.

439 mf, err = CreateMessageTemp(log, "account-test")

440 tcheck(t, err, "creating temp message file")

441 _, err = mf.Write(msgData)

442 tcheck(t, err, "write file")

443 defer CloseRemoveTempFile(log, mf, "temp message file")

444 m = Message{

445 Size: int64(len(msgData)),

446 }

447 err = acc.DeliverMailbox(log, "Inbox", "", &m, mf)

448 tcheck(t, err, "deliver mailbox")

449 if m.ID != msgFilesPerDir+1 {

450 t.Fatalf("got message id %d, expected $msgFilesPerDir+1", m.ID)

451 }

452

453 err = acc.Close()

454 tcheck(t, err, "closing account")

455 acc.WaitClosed()

456}

457

458func TestRemove(t *testing.T) {

459 log := mlog.New("store", nil)

460 os.RemoveAll("../testdata/store/data")

461 mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")

462 mox.MustLoadConfig(true, false)

463 err := Init(ctxbg)

464 tcheck(t, err, "init")

465 defer func() {

466 err := Close()

467 tcheck(t, err, "close")

468 }()

469 defer Switchboard()()

470

471 // Note: we are not removing the account from the config file. Nothing currently

472 // has a problem with that.

473

474 // Ensure account exists.

475 acc, err := OpenAccount(log, "mjl", false)

476 tcheck(t, err, "open account")

477

478 // Mark account removed. It will only be removed when we close the account.

479 err = acc.Remove(context.Background())

480 tcheck(t, err, "remove account")

481

482 p := filepath.Join(mox.DataDirPath("accounts"), "mjl")

483 _, err = os.Stat(p)

484 tcheck(t, err, "stat account dir")

485

486 err = acc.Close()

487 tcheck(t, err, "closing account")

488 acc.WaitClosed()

489 acc = nil

490

491 if _, err := os.Stat(p); err == nil || !errors.Is(err, fs.ErrNotExist) {

492 t.Fatalf(`got stat err %v for account directory, expected "does not exist"`, err)

493 }

494

495 // Recreate files and directories. We will reinitialize store/ without closing our

496 // account reference. This will apply the account removal. We only drop our (now

497 // broken) account reference when done.

498 acc, err = OpenAccount(log, "mjl", false)

499 tcheck(t, err, "open account")

500 defer func() {

501 acc.Close() // Ignore errors.

502 acc.WaitClosed()

503 CheckConsistencyOnClose = true

504 }()

505

506 // Init below will remove the directory, we are no longer consistent.

507 CheckConsistencyOnClose = false

508

509 err = acc.Remove(context.Background())

510 tcheck(t, err, "remove account")

511

512 _, err = os.Stat(p)

513 tcheck(t, err, "stat account dir")

514

515 err = Close()

516 tcheck(t, err, "close store")

517 err = Init(ctxbg)

518 tcheck(t, err, "init store")

519 if _, err := os.Stat(p); err == nil || !errors.Is(err, fs.ErrNotExist) {

520 t.Fatalf(`got stat err %v for account directory, expected "does not exist"`, err)

521 }

522 exists, err := bstore.QueryDB[AccountRemove](ctxbg, AuthDB).Exists()

523 tcheck(t, err, "checking for account removals")

524 tcompare(t, exists, false)

525}

526