1package store

2

3import (

4 "archive/tar"

5 "archive/zip"

6 "bufio"

7 "bytes"

8 "context"

9 "fmt"

10 "io"

11 "log/slog"

12 "os"

13 "path/filepath"

14 "sort"

15 "strings"

16 "time"

17

18 "github.com/mjl-/bstore"

19

20 "github.com/mjl-/mox/mlog"

21)

22

23// Archiver can archive multiple mailboxes and their messages.

24type Archiver interface {

25 // Add file to archive. If name ends with a slash, it is created as a directory and

26 // the returned io.WriteCloser can be ignored.

27 Create(name string, size int64, mtime time.Time) (io.WriteCloser, error)

28 Close() error

29}

30

31// TarArchiver is an Archiver that writes to a tar ifle.

32type TarArchiver struct {

33 *tar.Writer

34}

35

36// Create adds a file header to the tar file.

37func (a TarArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {

38 hdr := tar.Header{

39 Name: name,

40 Size: size,

41 Mode: 0660,

42 ModTime: mtime,

43 Format: tar.FormatPAX,

44 }

45 if err := a.WriteHeader(&hdr); err != nil {

46 return nil, err

47 }

48 return nopCloser{a}, nil

49}

50

51// ZipArchiver is an Archiver that writes to a zip file.

52type ZipArchiver struct {

53 *zip.Writer

54}

55

56// Create adds a file header to the zip file.

57func (a ZipArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {

58 hdr := zip.FileHeader{

59 Name: name,

60 Method: zip.Deflate,

61 Modified: mtime,

62 UncompressedSize64: uint64(size),

63 }

64 w, err := a.CreateHeader(&hdr)

65 if err != nil {

66 return nil, err

67 }

68 return nopCloser{w}, nil

69}

70

71type nopCloser struct {

72 io.Writer

73}

74

75// Close does nothing.

76func (nopCloser) Close() error {

77 return nil

78}

79

80// DirArchiver is an Archiver that writes to a directory.

81type DirArchiver struct {

82 Dir string

83}

84

85// Create create name in the file system, in dir.

86// name must always use forwarded slashes.

87func (a DirArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {

88 isdir := strings.HasSuffix(name, "/")

89 name = strings.TrimSuffix(name, "/")

90 p := filepath.Join(a.Dir, filepath.FromSlash(name))

91 os.MkdirAll(filepath.Dir(p), 0770)

92 if isdir {

93 return nil, os.Mkdir(p, 0770)

94 }

95 return os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0660)

96}

97

98// Close on a dir does nothing.

99func (a DirArchiver) Close() error {

100 return nil

101}

102

103// ExportMessages writes messages to archiver. Either in maildir format, or otherwise in

104// mbox. If mailboxOpt is empty, all mailboxes are exported, otherwise only the

105// named mailbox.

106//

107// Some errors are not fatal and result in skipped messages. In that happens, a

108// file "errors.txt" is added to the archive describing the errors. The goal is to

109// let users export (hopefully) most messages even in the face of errors.

110func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error {

111 // todo optimize: should prepare next file to add to archive (can be an mbox with many messages) while writing a file to the archive (which typically compresses, which takes time).

112

113 // Start transaction without closure, we are going to close it early, but don't

114 // want to deal with declaring many variables now to be able to assign them in a

115 // closure and use them afterwards.

116 tx, err := db.Begin(ctx, false)

117 if err != nil {

118 return fmt.Errorf("transaction: %v", err)

119 }

120 defer func() {

121 if tx != nil {

122 err := tx.Rollback()

123 log.Check(err, "transaction rollback after export error")

124 }

125 }()

126

127 start := time.Now()

128

129 // Set up mailbox names and ids.

130 id2name := map[int64]string{}

131 name2id := map[string]int64{}

132

133 mailboxes, err := bstore.QueryTx[Mailbox](tx).List()

134 if err != nil {

135 return fmt.Errorf("query mailboxes: %w", err)

136 }

137 for _, mb := range mailboxes {

138 id2name[mb.ID] = mb.Name

139 name2id[mb.Name] = mb.ID

140 }

141

142 var mailboxID int64

143 if mailboxOpt != "" {

144 var ok bool

145 mailboxID, ok = name2id[mailboxOpt]

146 if !ok {

147 return fmt.Errorf("mailbox not found")

148 }

149 }

150

151 var names []string

152 for _, name := range id2name {

153 if mailboxOpt != "" && name != mailboxOpt {

154 continue

155 }

156 names = append(names, name)

157 }

158 // We need to sort the names because maildirs can create subdirs. Ranging over

159 // id2name directly would randomize the directory names, we would create a sub

160 // maildir before the parent, and fail with "dir exists" when creating the parent

161 // dir.

162 sort.Slice(names, func(i, j int) bool {

163 return names[i] < names[j]

164 })

165

166 mailboxOrder := map[int64]int{}

167 for i, name := range names {

168 mbID := name2id[name]

169 mailboxOrder[mbID] = i

170 }

171

172 // Fetch all messages. This can take quite a bit of memory if the mailbox is large.

173 q := bstore.QueryTx[Message](tx)

174 if mailboxID > 0 {

175 q.FilterNonzero(Message{MailboxID: mailboxID})

176 }

177 msgs, err := q.List()

178 if err != nil {

179 return fmt.Errorf("listing messages: %v", err)

180 }

181

182 // Close transaction. We don't want to hold it for too long. We are now at risk

183 // that a message is be removed while we export, or flags changed. At least the

184 // size won't change. If we cannot open the message later on, we'll skip it and add

185 // an error message to an errors.txt file in the output archive.

186 if err := tx.Rollback(); err != nil {

187 return fmt.Errorf("closing transaction: %v", err)

188 }

189 tx = nil

190

191 // Order the messages by mailbox, received time and finally message ID.

192 sort.Slice(msgs, func(i, j int) bool {

193 iid := msgs[i].MailboxID

194 jid := msgs[j].MailboxID

195 if iid != jid {

196 return mailboxOrder[iid] < mailboxOrder[jid]

197 }

198 if !msgs[i].Received.Equal(msgs[j].Received) {

199 return msgs[i].Received.Before(msgs[j].Received)

200 }

201 return msgs[i].ID < msgs[j].ID

202 })

203

204 // We keep track of errors reading message files. We continue exporting and add an

205 // errors.txt file to the archive. In case of errors, the user can get (hopefully)

206 // most of their emails, and see something went wrong. For other errors, like

207 // writing to the archiver (e.g. a browser), we abort, because we don't want to

208 // continue with useless work.

209 var errors string

210

211 var curMailboxID int64 // Used to set curMailbox and finish a previous mbox file.

212 var curMailbox string

213

214 var mboxtmp *os.File

215 var mboxwriter *bufio.Writer

216 defer func() {

217 if mboxtmp != nil {

218 CloseRemoveTempFile(log, mboxtmp, "mbox")

219 }

220 }()

221

222 // For dovecot-keyword-style flags not in standard maildir.

223 maildirFlags := map[string]int{}

224 var maildirFlaglist []string

225 maildirFlag := func(flag string) string {

226 i, ok := maildirFlags[flag]

227 if !ok {

228 if len(maildirFlags) >= 26 {

229 // Max 26 flag characters.

230 return ""

231 }

232 i = len(maildirFlags)

233 maildirFlags[flag] = i

234 maildirFlaglist = append(maildirFlaglist, flag)

235 }

236 return string(rune('a' + i))

237 }

238

239 finishMailbox := func() error {

240 if maildir {

241 if len(maildirFlags) == 0 {

242 return nil

243 }

244

245 var b bytes.Buffer

246 for i, flag := range maildirFlaglist {

247 if _, err := fmt.Fprintf(&b, "%d %s\n", i, flag); err != nil {

248 return err

249 }

250 }

251 w, err := archiver.Create(curMailbox+"/dovecot-keywords", int64(b.Len()), start)

252 if err != nil {

253 return fmt.Errorf("adding dovecot-keywords: %v", err)

254 }

255 if _, err := w.Write(b.Bytes()); err != nil {

256 xerr := w.Close()

257 log.Check(xerr, "closing dovecot-keywords file after closing")

258 return fmt.Errorf("writing dovecot-keywords: %v", err)

259 }

260 maildirFlags = map[string]int{}

261 maildirFlaglist = nil

262 return w.Close()

263 }

264

265 if mboxtmp == nil {

266 return nil

267 }

268

269 if err := mboxwriter.Flush(); err != nil {

270 return fmt.Errorf("flush mbox writer: %v", err)

271 }

272 fi, err := mboxtmp.Stat()

273 if err != nil {

274 return fmt.Errorf("stat temporary mbox file: %v", err)

275 }

276 if _, err := mboxtmp.Seek(0, 0); err != nil {

277 return fmt.Errorf("seek to start of temporary mbox file")

278 }

279 w, err := archiver.Create(curMailbox+".mbox", fi.Size(), fi.ModTime())

280 if err != nil {

281 return fmt.Errorf("add mbox to archive: %v", err)

282 }

283 if _, err := io.Copy(w, mboxtmp); err != nil {

284 xerr := w.Close()

285 log.Check(xerr, "closing mbox message file after error")

286 return fmt.Errorf("copying temp mbox file to archive: %v", err)

287 }

288 if err := w.Close(); err != nil {

289 return fmt.Errorf("closing message file: %v", err)

290 }

291 name := mboxtmp.Name()

292 err = mboxtmp.Close()

293 log.Check(err, "closing temporary mbox file")

294 err = os.Remove(name)

295 log.Check(err, "removing temporary mbox file", slog.String("path", name))

296 mboxwriter = nil

297 mboxtmp = nil

298 return nil

299 }

300

301 exportMessage := func(m Message) error {

302 mp := filepath.Join(accountDir, "msg", MessagePath(m.ID))

303 var mr io.ReadCloser

304 if m.Size == int64(len(m.MsgPrefix)) {

305 mr = io.NopCloser(bytes.NewReader(m.MsgPrefix))

306 } else {

307 mf, err := os.Open(mp)

308 if err != nil {

309 errors += fmt.Sprintf("open message file for id %d, path %s: %v (message skipped)\n", m.ID, mp, err)

310 return nil

311 }

312 defer func() {

313 err := mf.Close()

314 log.Check(err, "closing message file after export")

315 }()

316 st, err := mf.Stat()

317 if err != nil {

318 errors += fmt.Sprintf("stat message file for id %d, path %s: %v (message skipped)\n", m.ID, mp, err)

319 return nil

320 }

321 size := st.Size() + int64(len(m.MsgPrefix))

322 if size != m.Size {

323 errors += fmt.Sprintf("message size mismatch for message id %d, database has %d, size is %d+%d=%d, using calculated size\n", m.ID, m.Size, len(m.MsgPrefix), st.Size(), size)

324 }

325 mr = FileMsgReader(m.MsgPrefix, mf)

326 }

327

328 if maildir {

329 p := curMailbox

330 if m.Flags.Seen {

331 p = filepath.Join(p, "cur")

332 } else {

333 p = filepath.Join(p, "new")

334 }

335 name := fmt.Sprintf("%d.%d.mox:2,", m.Received.Unix(), m.ID)

336

337 // Standard flags. May need to be sorted.

338 if m.Flags.Draft {

339 name += "D"

340 }

341 if m.Flags.Flagged {

342 name += "F"

343 }

344 if m.Flags.Answered {

345 name += "R"

346 }

347 if m.Flags.Seen {

348 name += "S"

349 }

350 if m.Flags.Deleted {

351 name += "T"

352 }

353

354 // Non-standard flag. We set them with a dovecot-keywords file.

355 if m.Flags.Forwarded {

356 name += maildirFlag("$Forwarded")

357 }

358 if m.Flags.Junk {

359 name += maildirFlag("$Junk")

360 }

361 if m.Flags.Notjunk {

362 name += maildirFlag("$NotJunk")

363 }

364 if m.Flags.Phishing {

365 name += maildirFlag("$Phishing")

366 }

367 if m.Flags.MDNSent {

368 name += maildirFlag("$MDNSent")

369 }

370

371 p = filepath.Join(p, name)

372

373 // We store messages with \r\n, maildir needs without. But we need to know the

374 // final size. So first convert, then create file with size, and write from buffer.

375 // todo: for large messages, we should go through a temporary file instead of memory.

376 var dst bytes.Buffer

377 r := bufio.NewReader(mr)

378 for {

379 line, rerr := r.ReadBytes('\n')

380 if rerr != io.EOF && rerr != nil {

381 errors += fmt.Sprintf("reading from message for id %d: %v (message skipped)\n", m.ID, err)

382 return nil

383 }

384 if len(line) > 0 {

385 if bytes.HasSuffix(line, []byte("\r\n")) {

386 line = line[:len(line)-1]

387 line[len(line)-1] = '\n'

388 }

389 if _, err = dst.Write(line); err != nil {

390 return fmt.Errorf("writing message: %v", err)

391 }

392 }

393 if rerr == io.EOF {

394 break

395 }

396 }

397 size := int64(dst.Len())

398 w, err := archiver.Create(p, size, m.Received)

399 if err != nil {

400 return fmt.Errorf("adding message to archive: %v", err)

401 }

402 if _, err := io.Copy(w, &dst); err != nil {

403 xerr := w.Close()

404 log.Check(xerr, "closing message")

405 return fmt.Errorf("copying message to archive: %v", err)

406 }

407 return w.Close()

408 }

409

410 mailfrom := "mox"

411 if m.MailFrom != "" {

412 mailfrom = m.MailFrom

413 }

414 if _, err := fmt.Fprintf(mboxwriter, "From %s %s\n", mailfrom, m.Received.Format(time.ANSIC)); err != nil {

415 return fmt.Errorf("write message line to mbox temp file: %v", err)

416 }

417

418 // Write message flags in the three headers that mbox consumers may (or may not) understand.

419 if m.Seen {

420 if _, err := fmt.Fprintf(mboxwriter, "Status: R\n"); err != nil {

421 return fmt.Errorf("writing status header: %v", err)

422 }

423 }

424 xstatus := ""

425 if m.Answered {

426 xstatus += "A"

427 }

428 if m.Flagged {

429 xstatus += "F"

430 }

431 if m.Draft {

432 xstatus += "T"

433 }

434 if m.Deleted {

435 xstatus += "D"

436 }

437 if xstatus != "" {

438 if _, err := fmt.Fprintf(mboxwriter, "X-Status: %s\n", xstatus); err != nil {

439 return fmt.Errorf("writing x-status header: %v", err)

440 }

441 }

442 var xkeywords []string

443 if m.Forwarded {

444 xkeywords = append(xkeywords, "$Forwarded")

445 }

446 if m.Junk && !m.Notjunk {

447 xkeywords = append(xkeywords, "$Junk")

448 }

449 if m.Notjunk && !m.Junk {

450 xkeywords = append(xkeywords, "$NotJunk")

451 }

452 if m.Phishing {

453 xkeywords = append(xkeywords, "$Phishing")

454 }

455 if m.MDNSent {

456 xkeywords = append(xkeywords, "$MDNSent")

457 }

458 if len(xkeywords) > 0 {

459 if _, err := fmt.Fprintf(mboxwriter, "X-Keywords: %s\n", strings.Join(xkeywords, ",")); err != nil {

460 return fmt.Errorf("writing x-keywords header: %v", err)

461 }

462 }

463

464 header := true

465 r := bufio.NewReader(mr)

466 for {

467 line, rerr := r.ReadBytes('\n')

468 if rerr != io.EOF && rerr != nil {

469 return fmt.Errorf("reading message: %v", err)

470 }

471 if len(line) > 0 {

472 if bytes.HasSuffix(line, []byte("\r\n")) {

473 line = line[:len(line)-1]

474 line[len(line)-1] = '\n'

475 }

476 if header && len(line) == 1 {

477 header = false

478 }

479 if header {

480 // Skip any previously stored flag-holding or now incorrect content-length headers.

481 // This assumes these headers are just a single line.

482 switch strings.ToLower(string(bytes.SplitN(line, []byte(":"), 2)[0])) {

483 case "status", "x-status", "x-keywords", "content-length":

484 continue

485 }

486 }

487 if bytes.HasPrefix(bytes.TrimLeft(line, ">"), []byte("From ")) {

488 if _, err := fmt.Fprint(mboxwriter, ">"); err != nil {

489 return fmt.Errorf("writing escaping >: %v", err)

490 }

491 }

492 if _, err := mboxwriter.Write(line); err != nil {

493 return fmt.Errorf("writing line: %v", err)

494 }

495 }

496 if rerr == io.EOF {

497 break

498 }

499 }

500 if _, err := fmt.Fprint(mboxwriter, "\n"); err != nil {

501 return fmt.Errorf("writing end of message newline: %v", err)

502 }

503 return nil

504 }

505

506 for _, m := range msgs {

507 if m.MailboxID != curMailboxID {

508 if err := finishMailbox(); err != nil {

509 return err

510 }

511

512 curMailbox = id2name[m.MailboxID]

513 curMailboxID = m.MailboxID

514 if maildir {

515 // Create the directories that show this is a maildir.

516 if _, err := archiver.Create(curMailbox+"/new/", 0, start); err != nil {

517 return fmt.Errorf("adding maildir new directory: %v", err)

518 }

519 if _, err := archiver.Create(curMailbox+"/cur/", 0, start); err != nil {

520 return fmt.Errorf("adding maildir cur directory: %v", err)

521 }

522 if _, err := archiver.Create(curMailbox+"/tmp/", 0, start); err != nil {

523 return fmt.Errorf("adding maildir tmp directory: %v", err)

524 }

525 } else {

526

527 mboxtmp, err = os.CreateTemp("", "mox-mail-export-mbox")

528 if err != nil {

529 return fmt.Errorf("creating temp mbox file: %v", err)

530 }

531 mboxwriter = bufio.NewWriter(mboxtmp)

532 }

533 }

534

535 if err := exportMessage(m); err != nil {

536 return err

537 }

538 }

539 if err := finishMailbox(); err != nil {

540 return err

541 }

542

543 if errors != "" {

544 w, err := archiver.Create("errors.txt", int64(len(errors)), time.Now())

545 if err != nil {

546 log.Errorx("adding errors.txt to archive", err)

547 return err

548 }

549 if _, err := w.Write([]byte(errors)); err != nil {

550 log.Errorx("writing errors.txt to archive", err)

551 xerr := w.Close()

552 log.Check(xerr, "closing errors.txt after error")

553 return err

554 }

555 if err := w.Close(); err != nil {

556 return err

557 }

558 }

559

560 return nil

561}

562