1 """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
2
3 # Notes for authors of new mailbox subclasses:
4 #
5 # Remember to fsync() changes to disk before closing a modified file
6 # or returning from a flush() method. See functions _sync_flush() and
7 # _sync_close().
8
9 import os
10 import time
11 import calendar
12 import socket
13 import errno
14 import copy
15 import warnings
16 import email
17 import email.message
18 import email.generator
19 import io
20 import contextlib
21 from types import GenericAlias
22 try:
23 import fcntl
24 except ImportError:
25 fcntl = None
26
27 __all__ = ['Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
28 'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
29 'BabylMessage', 'MMDFMessage', 'Error', 'NoSuchMailboxError',
30 'NotEmptyError', 'ExternalClashError', 'FormatError']
31
32 linesep = os.linesep.encode('ascii')
33
34 class ESC[4;38;5;81mMailbox:
35 """A group of messages in a particular place."""
36
37 def __init__(self, path, factory=None, create=True):
38 """Initialize a Mailbox instance."""
39 self._path = os.path.abspath(os.path.expanduser(path))
40 self._factory = factory
41
42 def add(self, message):
43 """Add message and return assigned key."""
44 raise NotImplementedError('Method must be implemented by subclass')
45
46 def remove(self, key):
47 """Remove the keyed message; raise KeyError if it doesn't exist."""
48 raise NotImplementedError('Method must be implemented by subclass')
49
50 def __delitem__(self, key):
51 self.remove(key)
52
53 def discard(self, key):
54 """If the keyed message exists, remove it."""
55 try:
56 self.remove(key)
57 except KeyError:
58 pass
59
60 def __setitem__(self, key, message):
61 """Replace the keyed message; raise KeyError if it doesn't exist."""
62 raise NotImplementedError('Method must be implemented by subclass')
63
64 def get(self, key, default=None):
65 """Return the keyed message, or default if it doesn't exist."""
66 try:
67 return self.__getitem__(key)
68 except KeyError:
69 return default
70
71 def __getitem__(self, key):
72 """Return the keyed message; raise KeyError if it doesn't exist."""
73 if not self._factory:
74 return self.get_message(key)
75 else:
76 with contextlib.closing(self.get_file(key)) as file:
77 return self._factory(file)
78
79 def get_message(self, key):
80 """Return a Message representation or raise a KeyError."""
81 raise NotImplementedError('Method must be implemented by subclass')
82
83 def get_string(self, key):
84 """Return a string representation or raise a KeyError.
85
86 Uses email.message.Message to create a 7bit clean string
87 representation of the message."""
88 return email.message_from_bytes(self.get_bytes(key)).as_string()
89
90 def get_bytes(self, key):
91 """Return a byte string representation or raise a KeyError."""
92 raise NotImplementedError('Method must be implemented by subclass')
93
94 def get_file(self, key):
95 """Return a file-like representation or raise a KeyError."""
96 raise NotImplementedError('Method must be implemented by subclass')
97
98 def iterkeys(self):
99 """Return an iterator over keys."""
100 raise NotImplementedError('Method must be implemented by subclass')
101
102 def keys(self):
103 """Return a list of keys."""
104 return list(self.iterkeys())
105
106 def itervalues(self):
107 """Return an iterator over all messages."""
108 for key in self.iterkeys():
109 try:
110 value = self[key]
111 except KeyError:
112 continue
113 yield value
114
115 def __iter__(self):
116 return self.itervalues()
117
118 def values(self):
119 """Return a list of messages. Memory intensive."""
120 return list(self.itervalues())
121
122 def iteritems(self):
123 """Return an iterator over (key, message) tuples."""
124 for key in self.iterkeys():
125 try:
126 value = self[key]
127 except KeyError:
128 continue
129 yield (key, value)
130
131 def items(self):
132 """Return a list of (key, message) tuples. Memory intensive."""
133 return list(self.iteritems())
134
135 def __contains__(self, key):
136 """Return True if the keyed message exists, False otherwise."""
137 raise NotImplementedError('Method must be implemented by subclass')
138
139 def __len__(self):
140 """Return a count of messages in the mailbox."""
141 raise NotImplementedError('Method must be implemented by subclass')
142
143 def clear(self):
144 """Delete all messages."""
145 for key in self.keys():
146 self.discard(key)
147
148 def pop(self, key, default=None):
149 """Delete the keyed message and return it, or default."""
150 try:
151 result = self[key]
152 except KeyError:
153 return default
154 self.discard(key)
155 return result
156
157 def popitem(self):
158 """Delete an arbitrary (key, message) pair and return it."""
159 for key in self.iterkeys():
160 return (key, self.pop(key)) # This is only run once.
161 else:
162 raise KeyError('No messages in mailbox')
163
164 def update(self, arg=None):
165 """Change the messages that correspond to certain keys."""
166 if hasattr(arg, 'iteritems'):
167 source = arg.iteritems()
168 elif hasattr(arg, 'items'):
169 source = arg.items()
170 else:
171 source = arg
172 bad_key = False
173 for key, message in source:
174 try:
175 self[key] = message
176 except KeyError:
177 bad_key = True
178 if bad_key:
179 raise KeyError('No message with key(s)')
180
181 def flush(self):
182 """Write any pending changes to the disk."""
183 raise NotImplementedError('Method must be implemented by subclass')
184
185 def lock(self):
186 """Lock the mailbox."""
187 raise NotImplementedError('Method must be implemented by subclass')
188
189 def unlock(self):
190 """Unlock the mailbox if it is locked."""
191 raise NotImplementedError('Method must be implemented by subclass')
192
193 def close(self):
194 """Flush and close the mailbox."""
195 raise NotImplementedError('Method must be implemented by subclass')
196
197 def _string_to_bytes(self, message):
198 # If a message is not 7bit clean, we refuse to handle it since it
199 # likely came from reading invalid messages in text mode, and that way
200 # lies mojibake.
201 try:
202 return message.encode('ascii')
203 except UnicodeError:
204 raise ValueError("String input must be ASCII-only; "
205 "use bytes or a Message instead")
206
207 # Whether each message must end in a newline
208 _append_newline = False
209
210 def _dump_message(self, message, target, mangle_from_=False):
211 # This assumes the target file is open in binary mode.
212 """Dump message contents to target file."""
213 if isinstance(message, email.message.Message):
214 buffer = io.BytesIO()
215 gen = email.generator.BytesGenerator(buffer, mangle_from_, 0)
216 gen.flatten(message)
217 buffer.seek(0)
218 data = buffer.read()
219 data = data.replace(b'\n', linesep)
220 target.write(data)
221 if self._append_newline and not data.endswith(linesep):
222 # Make sure the message ends with a newline
223 target.write(linesep)
224 elif isinstance(message, (str, bytes, io.StringIO)):
225 if isinstance(message, io.StringIO):
226 warnings.warn("Use of StringIO input is deprecated, "
227 "use BytesIO instead", DeprecationWarning, 3)
228 message = message.getvalue()
229 if isinstance(message, str):
230 message = self._string_to_bytes(message)
231 if mangle_from_:
232 message = message.replace(b'\nFrom ', b'\n>From ')
233 message = message.replace(b'\n', linesep)
234 target.write(message)
235 if self._append_newline and not message.endswith(linesep):
236 # Make sure the message ends with a newline
237 target.write(linesep)
238 elif hasattr(message, 'read'):
239 if hasattr(message, 'buffer'):
240 warnings.warn("Use of text mode files is deprecated, "
241 "use a binary mode file instead", DeprecationWarning, 3)
242 message = message.buffer
243 lastline = None
244 while True:
245 line = message.readline()
246 # Universal newline support.
247 if line.endswith(b'\r\n'):
248 line = line[:-2] + b'\n'
249 elif line.endswith(b'\r'):
250 line = line[:-1] + b'\n'
251 if not line:
252 break
253 if mangle_from_ and line.startswith(b'From '):
254 line = b'>From ' + line[5:]
255 line = line.replace(b'\n', linesep)
256 target.write(line)
257 lastline = line
258 if self._append_newline and lastline and not lastline.endswith(linesep):
259 # Make sure the message ends with a newline
260 target.write(linesep)
261 else:
262 raise TypeError('Invalid message type: %s' % type(message))
263
264 __class_getitem__ = classmethod(GenericAlias)
265
266
267 class ESC[4;38;5;81mMaildir(ESC[4;38;5;149mMailbox):
268 """A qmail-style Maildir mailbox."""
269
270 colon = ':'
271
272 def __init__(self, dirname, factory=None, create=True):
273 """Initialize a Maildir instance."""
274 Mailbox.__init__(self, dirname, factory, create)
275 self._paths = {
276 'tmp': os.path.join(self._path, 'tmp'),
277 'new': os.path.join(self._path, 'new'),
278 'cur': os.path.join(self._path, 'cur'),
279 }
280 if not os.path.exists(self._path):
281 if create:
282 os.mkdir(self._path, 0o700)
283 for path in self._paths.values():
284 os.mkdir(path, 0o700)
285 else:
286 raise NoSuchMailboxError(self._path)
287 self._toc = {}
288 self._toc_mtimes = {'cur': 0, 'new': 0}
289 self._last_read = 0 # Records last time we read cur/new
290 self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing
291
292 def add(self, message):
293 """Add message and return assigned key."""
294 tmp_file = self._create_tmp()
295 try:
296 self._dump_message(message, tmp_file)
297 except BaseException:
298 tmp_file.close()
299 os.remove(tmp_file.name)
300 raise
301 _sync_close(tmp_file)
302 if isinstance(message, MaildirMessage):
303 subdir = message.get_subdir()
304 suffix = self.colon + message.get_info()
305 if suffix == self.colon:
306 suffix = ''
307 else:
308 subdir = 'new'
309 suffix = ''
310 uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
311 dest = os.path.join(self._path, subdir, uniq + suffix)
312 if isinstance(message, MaildirMessage):
313 os.utime(tmp_file.name,
314 (os.path.getatime(tmp_file.name), message.get_date()))
315 # No file modification should be done after the file is moved to its
316 # final position in order to prevent race conditions with changes
317 # from other programs
318 try:
319 try:
320 os.link(tmp_file.name, dest)
321 except (AttributeError, PermissionError):
322 os.rename(tmp_file.name, dest)
323 else:
324 os.remove(tmp_file.name)
325 except OSError as e:
326 os.remove(tmp_file.name)
327 if e.errno == errno.EEXIST:
328 raise ExternalClashError('Name clash with existing message: %s'
329 % dest)
330 else:
331 raise
332 return uniq
333
334 def remove(self, key):
335 """Remove the keyed message; raise KeyError if it doesn't exist."""
336 os.remove(os.path.join(self._path, self._lookup(key)))
337
338 def discard(self, key):
339 """If the keyed message exists, remove it."""
340 # This overrides an inapplicable implementation in the superclass.
341 try:
342 self.remove(key)
343 except (KeyError, FileNotFoundError):
344 pass
345
346 def __setitem__(self, key, message):
347 """Replace the keyed message; raise KeyError if it doesn't exist."""
348 old_subpath = self._lookup(key)
349 temp_key = self.add(message)
350 temp_subpath = self._lookup(temp_key)
351 if isinstance(message, MaildirMessage):
352 # temp's subdir and suffix were specified by message.
353 dominant_subpath = temp_subpath
354 else:
355 # temp's subdir and suffix were defaults from add().
356 dominant_subpath = old_subpath
357 subdir = os.path.dirname(dominant_subpath)
358 if self.colon in dominant_subpath:
359 suffix = self.colon + dominant_subpath.split(self.colon)[-1]
360 else:
361 suffix = ''
362 self.discard(key)
363 tmp_path = os.path.join(self._path, temp_subpath)
364 new_path = os.path.join(self._path, subdir, key + suffix)
365 if isinstance(message, MaildirMessage):
366 os.utime(tmp_path,
367 (os.path.getatime(tmp_path), message.get_date()))
368 # No file modification should be done after the file is moved to its
369 # final position in order to prevent race conditions with changes
370 # from other programs
371 os.rename(tmp_path, new_path)
372
373 def get_message(self, key):
374 """Return a Message representation or raise a KeyError."""
375 subpath = self._lookup(key)
376 with open(os.path.join(self._path, subpath), 'rb') as f:
377 if self._factory:
378 msg = self._factory(f)
379 else:
380 msg = MaildirMessage(f)
381 subdir, name = os.path.split(subpath)
382 msg.set_subdir(subdir)
383 if self.colon in name:
384 msg.set_info(name.split(self.colon)[-1])
385 msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
386 return msg
387
388 def get_bytes(self, key):
389 """Return a bytes representation or raise a KeyError."""
390 with open(os.path.join(self._path, self._lookup(key)), 'rb') as f:
391 return f.read().replace(linesep, b'\n')
392
393 def get_file(self, key):
394 """Return a file-like representation or raise a KeyError."""
395 f = open(os.path.join(self._path, self._lookup(key)), 'rb')
396 return _ProxyFile(f)
397
398 def iterkeys(self):
399 """Return an iterator over keys."""
400 self._refresh()
401 for key in self._toc:
402 try:
403 self._lookup(key)
404 except KeyError:
405 continue
406 yield key
407
408 def __contains__(self, key):
409 """Return True if the keyed message exists, False otherwise."""
410 self._refresh()
411 return key in self._toc
412
413 def __len__(self):
414 """Return a count of messages in the mailbox."""
415 self._refresh()
416 return len(self._toc)
417
418 def flush(self):
419 """Write any pending changes to disk."""
420 # Maildir changes are always written immediately, so there's nothing
421 # to do.
422 pass
423
424 def lock(self):
425 """Lock the mailbox."""
426 return
427
428 def unlock(self):
429 """Unlock the mailbox if it is locked."""
430 return
431
432 def close(self):
433 """Flush and close the mailbox."""
434 return
435
436 def list_folders(self):
437 """Return a list of folder names."""
438 result = []
439 for entry in os.listdir(self._path):
440 if len(entry) > 1 and entry[0] == '.' and \
441 os.path.isdir(os.path.join(self._path, entry)):
442 result.append(entry[1:])
443 return result
444
445 def get_folder(self, folder):
446 """Return a Maildir instance for the named folder."""
447 return Maildir(os.path.join(self._path, '.' + folder),
448 factory=self._factory,
449 create=False)
450
451 def add_folder(self, folder):
452 """Create a folder and return a Maildir instance representing it."""
453 path = os.path.join(self._path, '.' + folder)
454 result = Maildir(path, factory=self._factory)
455 maildirfolder_path = os.path.join(path, 'maildirfolder')
456 if not os.path.exists(maildirfolder_path):
457 os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
458 0o666))
459 return result
460
461 def remove_folder(self, folder):
462 """Delete the named folder, which must be empty."""
463 path = os.path.join(self._path, '.' + folder)
464 for entry in os.listdir(os.path.join(path, 'new')) + \
465 os.listdir(os.path.join(path, 'cur')):
466 if len(entry) < 1 or entry[0] != '.':
467 raise NotEmptyError('Folder contains message(s): %s' % folder)
468 for entry in os.listdir(path):
469 if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
470 os.path.isdir(os.path.join(path, entry)):
471 raise NotEmptyError("Folder contains subdirectory '%s': %s" %
472 (folder, entry))
473 for root, dirs, files in os.walk(path, topdown=False):
474 for entry in files:
475 os.remove(os.path.join(root, entry))
476 for entry in dirs:
477 os.rmdir(os.path.join(root, entry))
478 os.rmdir(path)
479
480 def clean(self):
481 """Delete old files in "tmp"."""
482 now = time.time()
483 for entry in os.listdir(os.path.join(self._path, 'tmp')):
484 path = os.path.join(self._path, 'tmp', entry)
485 if now - os.path.getatime(path) > 129600: # 60 * 60 * 36
486 os.remove(path)
487
488 _count = 1 # This is used to generate unique file names.
489
490 def _create_tmp(self):
491 """Create a file in the tmp subdirectory and open and return it."""
492 now = time.time()
493 hostname = socket.gethostname()
494 if '/' in hostname:
495 hostname = hostname.replace('/', r'\057')
496 if ':' in hostname:
497 hostname = hostname.replace(':', r'\072')
498 uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
499 Maildir._count, hostname)
500 path = os.path.join(self._path, 'tmp', uniq)
501 try:
502 os.stat(path)
503 except FileNotFoundError:
504 Maildir._count += 1
505 try:
506 return _create_carefully(path)
507 except FileExistsError:
508 pass
509
510 # Fall through to here if stat succeeded or open raised EEXIST.
511 raise ExternalClashError('Name clash prevented file creation: %s' %
512 path)
513
514 def _refresh(self):
515 """Update table of contents mapping."""
516 # If it has been less than two seconds since the last _refresh() call,
517 # we have to unconditionally re-read the mailbox just in case it has
518 # been modified, because os.path.mtime() has a 2 sec resolution in the
519 # most common worst case (FAT) and a 1 sec resolution typically. This
520 # results in a few unnecessary re-reads when _refresh() is called
521 # multiple times in that interval, but once the clock ticks over, we
522 # will only re-read as needed. Because the filesystem might be being
523 # served by an independent system with its own clock, we record and
524 # compare with the mtimes from the filesystem. Because the other
525 # system's clock might be skewing relative to our clock, we add an
526 # extra delta to our wait. The default is one tenth second, but is an
527 # instance variable and so can be adjusted if dealing with a
528 # particularly skewed or irregular system.
529 if time.time() - self._last_read > 2 + self._skewfactor:
530 refresh = False
531 for subdir in self._toc_mtimes:
532 mtime = os.path.getmtime(self._paths[subdir])
533 if mtime > self._toc_mtimes[subdir]:
534 refresh = True
535 self._toc_mtimes[subdir] = mtime
536 if not refresh:
537 return
538 # Refresh toc
539 self._toc = {}
540 for subdir in self._toc_mtimes:
541 path = self._paths[subdir]
542 for entry in os.listdir(path):
543 p = os.path.join(path, entry)
544 if os.path.isdir(p):
545 continue
546 uniq = entry.split(self.colon)[0]
547 self._toc[uniq] = os.path.join(subdir, entry)
548 self._last_read = time.time()
549
550 def _lookup(self, key):
551 """Use TOC to return subpath for given key, or raise a KeyError."""
552 try:
553 if os.path.exists(os.path.join(self._path, self._toc[key])):
554 return self._toc[key]
555 except KeyError:
556 pass
557 self._refresh()
558 try:
559 return self._toc[key]
560 except KeyError:
561 raise KeyError('No message with key: %s' % key) from None
562
563 # This method is for backward compatibility only.
564 def next(self):
565 """Return the next message in a one-time iteration."""
566 if not hasattr(self, '_onetime_keys'):
567 self._onetime_keys = self.iterkeys()
568 while True:
569 try:
570 return self[next(self._onetime_keys)]
571 except StopIteration:
572 return None
573 except KeyError:
574 continue
575
576
577 class ESC[4;38;5;81m_singlefileMailbox(ESC[4;38;5;149mMailbox):
578 """A single-file mailbox."""
579
580 def __init__(self, path, factory=None, create=True):
581 """Initialize a single-file mailbox."""
582 Mailbox.__init__(self, path, factory, create)
583 try:
584 f = open(self._path, 'rb+')
585 except OSError as e:
586 if e.errno == errno.ENOENT:
587 if create:
588 f = open(self._path, 'wb+')
589 else:
590 raise NoSuchMailboxError(self._path)
591 elif e.errno in (errno.EACCES, errno.EROFS):
592 f = open(self._path, 'rb')
593 else:
594 raise
595 self._file = f
596 self._toc = None
597 self._next_key = 0
598 self._pending = False # No changes require rewriting the file.
599 self._pending_sync = False # No need to sync the file
600 self._locked = False
601 self._file_length = None # Used to record mailbox size
602
603 def add(self, message):
604 """Add message and return assigned key."""
605 self._lookup()
606 self._toc[self._next_key] = self._append_message(message)
607 self._next_key += 1
608 # _append_message appends the message to the mailbox file. We
609 # don't need a full rewrite + rename, sync is enough.
610 self._pending_sync = True
611 return self._next_key - 1
612
613 def remove(self, key):
614 """Remove the keyed message; raise KeyError if it doesn't exist."""
615 self._lookup(key)
616 del self._toc[key]
617 self._pending = True
618
619 def __setitem__(self, key, message):
620 """Replace the keyed message; raise KeyError if it doesn't exist."""
621 self._lookup(key)
622 self._toc[key] = self._append_message(message)
623 self._pending = True
624
625 def iterkeys(self):
626 """Return an iterator over keys."""
627 self._lookup()
628 yield from self._toc.keys()
629
630 def __contains__(self, key):
631 """Return True if the keyed message exists, False otherwise."""
632 self._lookup()
633 return key in self._toc
634
635 def __len__(self):
636 """Return a count of messages in the mailbox."""
637 self._lookup()
638 return len(self._toc)
639
640 def lock(self):
641 """Lock the mailbox."""
642 if not self._locked:
643 _lock_file(self._file)
644 self._locked = True
645
646 def unlock(self):
647 """Unlock the mailbox if it is locked."""
648 if self._locked:
649 _unlock_file(self._file)
650 self._locked = False
651
652 def flush(self):
653 """Write any pending changes to disk."""
654 if not self._pending:
655 if self._pending_sync:
656 # Messages have only been added, so syncing the file
657 # is enough.
658 _sync_flush(self._file)
659 self._pending_sync = False
660 return
661
662 # In order to be writing anything out at all, self._toc must
663 # already have been generated (and presumably has been modified
664 # by adding or deleting an item).
665 assert self._toc is not None
666
667 # Check length of self._file; if it's changed, some other process
668 # has modified the mailbox since we scanned it.
669 self._file.seek(0, 2)
670 cur_len = self._file.tell()
671 if cur_len != self._file_length:
672 raise ExternalClashError('Size of mailbox file changed '
673 '(expected %i, found %i)' %
674 (self._file_length, cur_len))
675
676 new_file = _create_temporary(self._path)
677 try:
678 new_toc = {}
679 self._pre_mailbox_hook(new_file)
680 for key in sorted(self._toc.keys()):
681 start, stop = self._toc[key]
682 self._file.seek(start)
683 self._pre_message_hook(new_file)
684 new_start = new_file.tell()
685 while True:
686 buffer = self._file.read(min(4096,
687 stop - self._file.tell()))
688 if not buffer:
689 break
690 new_file.write(buffer)
691 new_toc[key] = (new_start, new_file.tell())
692 self._post_message_hook(new_file)
693 self._file_length = new_file.tell()
694 except:
695 new_file.close()
696 os.remove(new_file.name)
697 raise
698 _sync_close(new_file)
699 # self._file is about to get replaced, so no need to sync.
700 self._file.close()
701 # Make sure the new file's mode is the same as the old file's
702 mode = os.stat(self._path).st_mode
703 os.chmod(new_file.name, mode)
704 try:
705 os.rename(new_file.name, self._path)
706 except FileExistsError:
707 os.remove(self._path)
708 os.rename(new_file.name, self._path)
709 self._file = open(self._path, 'rb+')
710 self._toc = new_toc
711 self._pending = False
712 self._pending_sync = False
713 if self._locked:
714 _lock_file(self._file, dotlock=False)
715
716 def _pre_mailbox_hook(self, f):
717 """Called before writing the mailbox to file f."""
718 return
719
720 def _pre_message_hook(self, f):
721 """Called before writing each message to file f."""
722 return
723
724 def _post_message_hook(self, f):
725 """Called after writing each message to file f."""
726 return
727
728 def close(self):
729 """Flush and close the mailbox."""
730 try:
731 self.flush()
732 finally:
733 try:
734 if self._locked:
735 self.unlock()
736 finally:
737 self._file.close() # Sync has been done by self.flush() above.
738
739 def _lookup(self, key=None):
740 """Return (start, stop) or raise KeyError."""
741 if self._toc is None:
742 self._generate_toc()
743 if key is not None:
744 try:
745 return self._toc[key]
746 except KeyError:
747 raise KeyError('No message with key: %s' % key) from None
748
749 def _append_message(self, message):
750 """Append message to mailbox and return (start, stop) offsets."""
751 self._file.seek(0, 2)
752 before = self._file.tell()
753 if len(self._toc) == 0 and not self._pending:
754 # This is the first message, and the _pre_mailbox_hook
755 # hasn't yet been called. If self._pending is True,
756 # messages have been removed, so _pre_mailbox_hook must
757 # have been called already.
758 self._pre_mailbox_hook(self._file)
759 try:
760 self._pre_message_hook(self._file)
761 offsets = self._install_message(message)
762 self._post_message_hook(self._file)
763 except BaseException:
764 self._file.truncate(before)
765 raise
766 self._file.flush()
767 self._file_length = self._file.tell() # Record current length of mailbox
768 return offsets
769
770
771
772 class ESC[4;38;5;81m_mboxMMDF(ESC[4;38;5;149m_singlefileMailbox):
773 """An mbox or MMDF mailbox."""
774
775 _mangle_from_ = True
776
777 def get_message(self, key):
778 """Return a Message representation or raise a KeyError."""
779 start, stop = self._lookup(key)
780 self._file.seek(start)
781 from_line = self._file.readline().replace(linesep, b'')
782 string = self._file.read(stop - self._file.tell())
783 msg = self._message_factory(string.replace(linesep, b'\n'))
784 msg.set_from(from_line[5:].decode('ascii'))
785 return msg
786
787 def get_string(self, key, from_=False):
788 """Return a string representation or raise a KeyError."""
789 return email.message_from_bytes(
790 self.get_bytes(key, from_)).as_string(unixfrom=from_)
791
792 def get_bytes(self, key, from_=False):
793 """Return a string representation or raise a KeyError."""
794 start, stop = self._lookup(key)
795 self._file.seek(start)
796 if not from_:
797 self._file.readline()
798 string = self._file.read(stop - self._file.tell())
799 return string.replace(linesep, b'\n')
800
801 def get_file(self, key, from_=False):
802 """Return a file-like representation or raise a KeyError."""
803 start, stop = self._lookup(key)
804 self._file.seek(start)
805 if not from_:
806 self._file.readline()
807 return _PartialFile(self._file, self._file.tell(), stop)
808
809 def _install_message(self, message):
810 """Format a message and blindly write to self._file."""
811 from_line = None
812 if isinstance(message, str):
813 message = self._string_to_bytes(message)
814 if isinstance(message, bytes) and message.startswith(b'From '):
815 newline = message.find(b'\n')
816 if newline != -1:
817 from_line = message[:newline]
818 message = message[newline + 1:]
819 else:
820 from_line = message
821 message = b''
822 elif isinstance(message, _mboxMMDFMessage):
823 author = message.get_from().encode('ascii')
824 from_line = b'From ' + author
825 elif isinstance(message, email.message.Message):
826 from_line = message.get_unixfrom() # May be None.
827 if from_line is not None:
828 from_line = from_line.encode('ascii')
829 if from_line is None:
830 from_line = b'From MAILER-DAEMON ' + time.asctime(time.gmtime()).encode()
831 start = self._file.tell()
832 self._file.write(from_line + linesep)
833 self._dump_message(message, self._file, self._mangle_from_)
834 stop = self._file.tell()
835 return (start, stop)
836
837
838 class ESC[4;38;5;81mmbox(ESC[4;38;5;149m_mboxMMDF):
839 """A classic mbox mailbox."""
840
841 _mangle_from_ = True
842
843 # All messages must end in a newline character, and
844 # _post_message_hooks outputs an empty line between messages.
845 _append_newline = True
846
847 def __init__(self, path, factory=None, create=True):
848 """Initialize an mbox mailbox."""
849 self._message_factory = mboxMessage
850 _mboxMMDF.__init__(self, path, factory, create)
851
852 def _post_message_hook(self, f):
853 """Called after writing each message to file f."""
854 f.write(linesep)
855
856 def _generate_toc(self):
857 """Generate key-to-(start, stop) table of contents."""
858 starts, stops = [], []
859 last_was_empty = False
860 self._file.seek(0)
861 while True:
862 line_pos = self._file.tell()
863 line = self._file.readline()
864 if line.startswith(b'From '):
865 if len(stops) < len(starts):
866 if last_was_empty:
867 stops.append(line_pos - len(linesep))
868 else:
869 # The last line before the "From " line wasn't
870 # blank, but we consider it a start of a
871 # message anyway.
872 stops.append(line_pos)
873 starts.append(line_pos)
874 last_was_empty = False
875 elif not line:
876 if last_was_empty:
877 stops.append(line_pos - len(linesep))
878 else:
879 stops.append(line_pos)
880 break
881 elif line == linesep:
882 last_was_empty = True
883 else:
884 last_was_empty = False
885 self._toc = dict(enumerate(zip(starts, stops)))
886 self._next_key = len(self._toc)
887 self._file_length = self._file.tell()
888
889
890 class ESC[4;38;5;81mMMDF(ESC[4;38;5;149m_mboxMMDF):
891 """An MMDF mailbox."""
892
893 def __init__(self, path, factory=None, create=True):
894 """Initialize an MMDF mailbox."""
895 self._message_factory = MMDFMessage
896 _mboxMMDF.__init__(self, path, factory, create)
897
898 def _pre_message_hook(self, f):
899 """Called before writing each message to file f."""
900 f.write(b'\001\001\001\001' + linesep)
901
902 def _post_message_hook(self, f):
903 """Called after writing each message to file f."""
904 f.write(linesep + b'\001\001\001\001' + linesep)
905
906 def _generate_toc(self):
907 """Generate key-to-(start, stop) table of contents."""
908 starts, stops = [], []
909 self._file.seek(0)
910 next_pos = 0
911 while True:
912 line_pos = next_pos
913 line = self._file.readline()
914 next_pos = self._file.tell()
915 if line.startswith(b'\001\001\001\001' + linesep):
916 starts.append(next_pos)
917 while True:
918 line_pos = next_pos
919 line = self._file.readline()
920 next_pos = self._file.tell()
921 if line == b'\001\001\001\001' + linesep:
922 stops.append(line_pos - len(linesep))
923 break
924 elif not line:
925 stops.append(line_pos)
926 break
927 elif not line:
928 break
929 self._toc = dict(enumerate(zip(starts, stops)))
930 self._next_key = len(self._toc)
931 self._file.seek(0, 2)
932 self._file_length = self._file.tell()
933
934
935 class ESC[4;38;5;81mMH(ESC[4;38;5;149mMailbox):
936 """An MH mailbox."""
937
938 def __init__(self, path, factory=None, create=True):
939 """Initialize an MH instance."""
940 Mailbox.__init__(self, path, factory, create)
941 if not os.path.exists(self._path):
942 if create:
943 os.mkdir(self._path, 0o700)
944 os.close(os.open(os.path.join(self._path, '.mh_sequences'),
945 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600))
946 else:
947 raise NoSuchMailboxError(self._path)
948 self._locked = False
949
950 def add(self, message):
951 """Add message and return assigned key."""
952 keys = self.keys()
953 if len(keys) == 0:
954 new_key = 1
955 else:
956 new_key = max(keys) + 1
957 new_path = os.path.join(self._path, str(new_key))
958 f = _create_carefully(new_path)
959 closed = False
960 try:
961 if self._locked:
962 _lock_file(f)
963 try:
964 try:
965 self._dump_message(message, f)
966 except BaseException:
967 # Unlock and close so it can be deleted on Windows
968 if self._locked:
969 _unlock_file(f)
970 _sync_close(f)
971 closed = True
972 os.remove(new_path)
973 raise
974 if isinstance(message, MHMessage):
975 self._dump_sequences(message, new_key)
976 finally:
977 if self._locked:
978 _unlock_file(f)
979 finally:
980 if not closed:
981 _sync_close(f)
982 return new_key
983
984 def remove(self, key):
985 """Remove the keyed message; raise KeyError if it doesn't exist."""
986 path = os.path.join(self._path, str(key))
987 try:
988 f = open(path, 'rb+')
989 except OSError as e:
990 if e.errno == errno.ENOENT:
991 raise KeyError('No message with key: %s' % key)
992 else:
993 raise
994 else:
995 f.close()
996 os.remove(path)
997
998 def __setitem__(self, key, message):
999 """Replace the keyed message; raise KeyError if it doesn't exist."""
1000 path = os.path.join(self._path, str(key))
1001 try:
1002 f = open(path, 'rb+')
1003 except OSError as e:
1004 if e.errno == errno.ENOENT:
1005 raise KeyError('No message with key: %s' % key)
1006 else:
1007 raise
1008 try:
1009 if self._locked:
1010 _lock_file(f)
1011 try:
1012 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
1013 self._dump_message(message, f)
1014 if isinstance(message, MHMessage):
1015 self._dump_sequences(message, key)
1016 finally:
1017 if self._locked:
1018 _unlock_file(f)
1019 finally:
1020 _sync_close(f)
1021
1022 def get_message(self, key):
1023 """Return a Message representation or raise a KeyError."""
1024 try:
1025 if self._locked:
1026 f = open(os.path.join(self._path, str(key)), 'rb+')
1027 else:
1028 f = open(os.path.join(self._path, str(key)), 'rb')
1029 except OSError as e:
1030 if e.errno == errno.ENOENT:
1031 raise KeyError('No message with key: %s' % key)
1032 else:
1033 raise
1034 with f:
1035 if self._locked:
1036 _lock_file(f)
1037 try:
1038 msg = MHMessage(f)
1039 finally:
1040 if self._locked:
1041 _unlock_file(f)
1042 for name, key_list in self.get_sequences().items():
1043 if key in key_list:
1044 msg.add_sequence(name)
1045 return msg
1046
1047 def get_bytes(self, key):
1048 """Return a bytes representation or raise a KeyError."""
1049 try:
1050 if self._locked:
1051 f = open(os.path.join(self._path, str(key)), 'rb+')
1052 else:
1053 f = open(os.path.join(self._path, str(key)), 'rb')
1054 except OSError as e:
1055 if e.errno == errno.ENOENT:
1056 raise KeyError('No message with key: %s' % key)
1057 else:
1058 raise
1059 with f:
1060 if self._locked:
1061 _lock_file(f)
1062 try:
1063 return f.read().replace(linesep, b'\n')
1064 finally:
1065 if self._locked:
1066 _unlock_file(f)
1067
1068 def get_file(self, key):
1069 """Return a file-like representation or raise a KeyError."""
1070 try:
1071 f = open(os.path.join(self._path, str(key)), 'rb')
1072 except OSError as e:
1073 if e.errno == errno.ENOENT:
1074 raise KeyError('No message with key: %s' % key)
1075 else:
1076 raise
1077 return _ProxyFile(f)
1078
1079 def iterkeys(self):
1080 """Return an iterator over keys."""
1081 return iter(sorted(int(entry) for entry in os.listdir(self._path)
1082 if entry.isdigit()))
1083
1084 def __contains__(self, key):
1085 """Return True if the keyed message exists, False otherwise."""
1086 return os.path.exists(os.path.join(self._path, str(key)))
1087
1088 def __len__(self):
1089 """Return a count of messages in the mailbox."""
1090 return len(list(self.iterkeys()))
1091
1092 def lock(self):
1093 """Lock the mailbox."""
1094 if not self._locked:
1095 self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
1096 _lock_file(self._file)
1097 self._locked = True
1098
1099 def unlock(self):
1100 """Unlock the mailbox if it is locked."""
1101 if self._locked:
1102 _unlock_file(self._file)
1103 _sync_close(self._file)
1104 del self._file
1105 self._locked = False
1106
1107 def flush(self):
1108 """Write any pending changes to the disk."""
1109 return
1110
1111 def close(self):
1112 """Flush and close the mailbox."""
1113 if self._locked:
1114 self.unlock()
1115
1116 def list_folders(self):
1117 """Return a list of folder names."""
1118 result = []
1119 for entry in os.listdir(self._path):
1120 if os.path.isdir(os.path.join(self._path, entry)):
1121 result.append(entry)
1122 return result
1123
1124 def get_folder(self, folder):
1125 """Return an MH instance for the named folder."""
1126 return MH(os.path.join(self._path, folder),
1127 factory=self._factory, create=False)
1128
1129 def add_folder(self, folder):
1130 """Create a folder and return an MH instance representing it."""
1131 return MH(os.path.join(self._path, folder),
1132 factory=self._factory)
1133
1134 def remove_folder(self, folder):
1135 """Delete the named folder, which must be empty."""
1136 path = os.path.join(self._path, folder)
1137 entries = os.listdir(path)
1138 if entries == ['.mh_sequences']:
1139 os.remove(os.path.join(path, '.mh_sequences'))
1140 elif entries == []:
1141 pass
1142 else:
1143 raise NotEmptyError('Folder not empty: %s' % self._path)
1144 os.rmdir(path)
1145
1146 def get_sequences(self):
1147 """Return a name-to-key-list dictionary to define each sequence."""
1148 results = {}
1149 with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f:
1150 all_keys = set(self.keys())
1151 for line in f:
1152 try:
1153 name, contents = line.split(':')
1154 keys = set()
1155 for spec in contents.split():
1156 if spec.isdigit():
1157 keys.add(int(spec))
1158 else:
1159 start, stop = (int(x) for x in spec.split('-'))
1160 keys.update(range(start, stop + 1))
1161 results[name] = [key for key in sorted(keys) \
1162 if key in all_keys]
1163 if len(results[name]) == 0:
1164 del results[name]
1165 except ValueError:
1166 raise FormatError('Invalid sequence specification: %s' %
1167 line.rstrip())
1168 return results
1169
1170 def set_sequences(self, sequences):
1171 """Set sequences using the given name-to-key-list dictionary."""
1172 f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII')
1173 try:
1174 os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
1175 for name, keys in sequences.items():
1176 if len(keys) == 0:
1177 continue
1178 f.write(name + ':')
1179 prev = None
1180 completing = False
1181 for key in sorted(set(keys)):
1182 if key - 1 == prev:
1183 if not completing:
1184 completing = True
1185 f.write('-')
1186 elif completing:
1187 completing = False
1188 f.write('%s %s' % (prev, key))
1189 else:
1190 f.write(' %s' % key)
1191 prev = key
1192 if completing:
1193 f.write(str(prev) + '\n')
1194 else:
1195 f.write('\n')
1196 finally:
1197 _sync_close(f)
1198
1199 def pack(self):
1200 """Re-name messages to eliminate numbering gaps. Invalidates keys."""
1201 sequences = self.get_sequences()
1202 prev = 0
1203 changes = []
1204 for key in self.iterkeys():
1205 if key - 1 != prev:
1206 changes.append((key, prev + 1))
1207 try:
1208 os.link(os.path.join(self._path, str(key)),
1209 os.path.join(self._path, str(prev + 1)))
1210 except (AttributeError, PermissionError):
1211 os.rename(os.path.join(self._path, str(key)),
1212 os.path.join(self._path, str(prev + 1)))
1213 else:
1214 os.unlink(os.path.join(self._path, str(key)))
1215 prev += 1
1216 self._next_key = prev + 1
1217 if len(changes) == 0:
1218 return
1219 for name, key_list in sequences.items():
1220 for old, new in changes:
1221 if old in key_list:
1222 key_list[key_list.index(old)] = new
1223 self.set_sequences(sequences)
1224
1225 def _dump_sequences(self, message, key):
1226 """Inspect a new MHMessage and update sequences appropriately."""
1227 pending_sequences = message.get_sequences()
1228 all_sequences = self.get_sequences()
1229 for name, key_list in all_sequences.items():
1230 if name in pending_sequences:
1231 key_list.append(key)
1232 elif key in key_list:
1233 del key_list[key_list.index(key)]
1234 for sequence in pending_sequences:
1235 if sequence not in all_sequences:
1236 all_sequences[sequence] = [key]
1237 self.set_sequences(all_sequences)
1238
1239
1240 class ESC[4;38;5;81mBabyl(ESC[4;38;5;149m_singlefileMailbox):
1241 """An Rmail-style Babyl mailbox."""
1242
1243 _special_labels = frozenset({'unseen', 'deleted', 'filed', 'answered',
1244 'forwarded', 'edited', 'resent'})
1245
1246 def __init__(self, path, factory=None, create=True):
1247 """Initialize a Babyl mailbox."""
1248 _singlefileMailbox.__init__(self, path, factory, create)
1249 self._labels = {}
1250
1251 def add(self, message):
1252 """Add message and return assigned key."""
1253 key = _singlefileMailbox.add(self, message)
1254 if isinstance(message, BabylMessage):
1255 self._labels[key] = message.get_labels()
1256 return key
1257
1258 def remove(self, key):
1259 """Remove the keyed message; raise KeyError if it doesn't exist."""
1260 _singlefileMailbox.remove(self, key)
1261 if key in self._labels:
1262 del self._labels[key]
1263
1264 def __setitem__(self, key, message):
1265 """Replace the keyed message; raise KeyError if it doesn't exist."""
1266 _singlefileMailbox.__setitem__(self, key, message)
1267 if isinstance(message, BabylMessage):
1268 self._labels[key] = message.get_labels()
1269
1270 def get_message(self, key):
1271 """Return a Message representation or raise a KeyError."""
1272 start, stop = self._lookup(key)
1273 self._file.seek(start)
1274 self._file.readline() # Skip b'1,' line specifying labels.
1275 original_headers = io.BytesIO()
1276 while True:
1277 line = self._file.readline()
1278 if line == b'*** EOOH ***' + linesep or not line:
1279 break
1280 original_headers.write(line.replace(linesep, b'\n'))
1281 visible_headers = io.BytesIO()
1282 while True:
1283 line = self._file.readline()
1284 if line == linesep or not line:
1285 break
1286 visible_headers.write(line.replace(linesep, b'\n'))
1287 # Read up to the stop, or to the end
1288 n = stop - self._file.tell()
1289 assert n >= 0
1290 body = self._file.read(n)
1291 body = body.replace(linesep, b'\n')
1292 msg = BabylMessage(original_headers.getvalue() + body)
1293 msg.set_visible(visible_headers.getvalue())
1294 if key in self._labels:
1295 msg.set_labels(self._labels[key])
1296 return msg
1297
1298 def get_bytes(self, key):
1299 """Return a string representation or raise a KeyError."""
1300 start, stop = self._lookup(key)
1301 self._file.seek(start)
1302 self._file.readline() # Skip b'1,' line specifying labels.
1303 original_headers = io.BytesIO()
1304 while True:
1305 line = self._file.readline()
1306 if line == b'*** EOOH ***' + linesep or not line:
1307 break
1308 original_headers.write(line.replace(linesep, b'\n'))
1309 while True:
1310 line = self._file.readline()
1311 if line == linesep or not line:
1312 break
1313 headers = original_headers.getvalue()
1314 n = stop - self._file.tell()
1315 assert n >= 0
1316 data = self._file.read(n)
1317 data = data.replace(linesep, b'\n')
1318 return headers + data
1319
1320 def get_file(self, key):
1321 """Return a file-like representation or raise a KeyError."""
1322 return io.BytesIO(self.get_bytes(key).replace(b'\n', linesep))
1323
1324 def get_labels(self):
1325 """Return a list of user-defined labels in the mailbox."""
1326 self._lookup()
1327 labels = set()
1328 for label_list in self._labels.values():
1329 labels.update(label_list)
1330 labels.difference_update(self._special_labels)
1331 return list(labels)
1332
1333 def _generate_toc(self):
1334 """Generate key-to-(start, stop) table of contents."""
1335 starts, stops = [], []
1336 self._file.seek(0)
1337 next_pos = 0
1338 label_lists = []
1339 while True:
1340 line_pos = next_pos
1341 line = self._file.readline()
1342 next_pos = self._file.tell()
1343 if line == b'\037\014' + linesep:
1344 if len(stops) < len(starts):
1345 stops.append(line_pos - len(linesep))
1346 starts.append(next_pos)
1347 labels = [label.strip() for label
1348 in self._file.readline()[1:].split(b',')
1349 if label.strip()]
1350 label_lists.append(labels)
1351 elif line == b'\037' or line == b'\037' + linesep:
1352 if len(stops) < len(starts):
1353 stops.append(line_pos - len(linesep))
1354 elif not line:
1355 stops.append(line_pos - len(linesep))
1356 break
1357 self._toc = dict(enumerate(zip(starts, stops)))
1358 self._labels = dict(enumerate(label_lists))
1359 self._next_key = len(self._toc)
1360 self._file.seek(0, 2)
1361 self._file_length = self._file.tell()
1362
1363 def _pre_mailbox_hook(self, f):
1364 """Called before writing the mailbox to file f."""
1365 babyl = b'BABYL OPTIONS:' + linesep
1366 babyl += b'Version: 5' + linesep
1367 labels = self.get_labels()
1368 labels = (label.encode() for label in labels)
1369 babyl += b'Labels:' + b','.join(labels) + linesep
1370 babyl += b'\037'
1371 f.write(babyl)
1372
1373 def _pre_message_hook(self, f):
1374 """Called before writing each message to file f."""
1375 f.write(b'\014' + linesep)
1376
1377 def _post_message_hook(self, f):
1378 """Called after writing each message to file f."""
1379 f.write(linesep + b'\037')
1380
1381 def _install_message(self, message):
1382 """Write message contents and return (start, stop)."""
1383 start = self._file.tell()
1384 if isinstance(message, BabylMessage):
1385 special_labels = []
1386 labels = []
1387 for label in message.get_labels():
1388 if label in self._special_labels:
1389 special_labels.append(label)
1390 else:
1391 labels.append(label)
1392 self._file.write(b'1')
1393 for label in special_labels:
1394 self._file.write(b', ' + label.encode())
1395 self._file.write(b',,')
1396 for label in labels:
1397 self._file.write(b' ' + label.encode() + b',')
1398 self._file.write(linesep)
1399 else:
1400 self._file.write(b'1,,' + linesep)
1401 if isinstance(message, email.message.Message):
1402 orig_buffer = io.BytesIO()
1403 orig_generator = email.generator.BytesGenerator(orig_buffer, False, 0)
1404 orig_generator.flatten(message)
1405 orig_buffer.seek(0)
1406 while True:
1407 line = orig_buffer.readline()
1408 self._file.write(line.replace(b'\n', linesep))
1409 if line == b'\n' or not line:
1410 break
1411 self._file.write(b'*** EOOH ***' + linesep)
1412 if isinstance(message, BabylMessage):
1413 vis_buffer = io.BytesIO()
1414 vis_generator = email.generator.BytesGenerator(vis_buffer, False, 0)
1415 vis_generator.flatten(message.get_visible())
1416 while True:
1417 line = vis_buffer.readline()
1418 self._file.write(line.replace(b'\n', linesep))
1419 if line == b'\n' or not line:
1420 break
1421 else:
1422 orig_buffer.seek(0)
1423 while True:
1424 line = orig_buffer.readline()
1425 self._file.write(line.replace(b'\n', linesep))
1426 if line == b'\n' or not line:
1427 break
1428 while True:
1429 buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
1430 if not buffer:
1431 break
1432 self._file.write(buffer.replace(b'\n', linesep))
1433 elif isinstance(message, (bytes, str, io.StringIO)):
1434 if isinstance(message, io.StringIO):
1435 warnings.warn("Use of StringIO input is deprecated, "
1436 "use BytesIO instead", DeprecationWarning, 3)
1437 message = message.getvalue()
1438 if isinstance(message, str):
1439 message = self._string_to_bytes(message)
1440 body_start = message.find(b'\n\n') + 2
1441 if body_start - 2 != -1:
1442 self._file.write(message[:body_start].replace(b'\n', linesep))
1443 self._file.write(b'*** EOOH ***' + linesep)
1444 self._file.write(message[:body_start].replace(b'\n', linesep))
1445 self._file.write(message[body_start:].replace(b'\n', linesep))
1446 else:
1447 self._file.write(b'*** EOOH ***' + linesep + linesep)
1448 self._file.write(message.replace(b'\n', linesep))
1449 elif hasattr(message, 'readline'):
1450 if hasattr(message, 'buffer'):
1451 warnings.warn("Use of text mode files is deprecated, "
1452 "use a binary mode file instead", DeprecationWarning, 3)
1453 message = message.buffer
1454 original_pos = message.tell()
1455 first_pass = True
1456 while True:
1457 line = message.readline()
1458 # Universal newline support.
1459 if line.endswith(b'\r\n'):
1460 line = line[:-2] + b'\n'
1461 elif line.endswith(b'\r'):
1462 line = line[:-1] + b'\n'
1463 self._file.write(line.replace(b'\n', linesep))
1464 if line == b'\n' or not line:
1465 if first_pass:
1466 first_pass = False
1467 self._file.write(b'*** EOOH ***' + linesep)
1468 message.seek(original_pos)
1469 else:
1470 break
1471 while True:
1472 line = message.readline()
1473 if not line:
1474 break
1475 # Universal newline support.
1476 if line.endswith(b'\r\n'):
1477 line = line[:-2] + linesep
1478 elif line.endswith(b'\r'):
1479 line = line[:-1] + linesep
1480 elif line.endswith(b'\n'):
1481 line = line[:-1] + linesep
1482 self._file.write(line)
1483 else:
1484 raise TypeError('Invalid message type: %s' % type(message))
1485 stop = self._file.tell()
1486 return (start, stop)
1487
1488
1489 class ESC[4;38;5;81mMessage(ESC[4;38;5;149memailESC[4;38;5;149m.ESC[4;38;5;149mmessageESC[4;38;5;149m.ESC[4;38;5;149mMessage):
1490 """Message with mailbox-format-specific properties."""
1491
1492 def __init__(self, message=None):
1493 """Initialize a Message instance."""
1494 if isinstance(message, email.message.Message):
1495 self._become_message(copy.deepcopy(message))
1496 if isinstance(message, Message):
1497 message._explain_to(self)
1498 elif isinstance(message, bytes):
1499 self._become_message(email.message_from_bytes(message))
1500 elif isinstance(message, str):
1501 self._become_message(email.message_from_string(message))
1502 elif isinstance(message, io.TextIOWrapper):
1503 self._become_message(email.message_from_file(message))
1504 elif hasattr(message, "read"):
1505 self._become_message(email.message_from_binary_file(message))
1506 elif message is None:
1507 email.message.Message.__init__(self)
1508 else:
1509 raise TypeError('Invalid message type: %s' % type(message))
1510
1511 def _become_message(self, message):
1512 """Assume the non-format-specific state of message."""
1513 type_specific = getattr(message, '_type_specific_attributes', [])
1514 for name in message.__dict__:
1515 if name not in type_specific:
1516 self.__dict__[name] = message.__dict__[name]
1517
1518 def _explain_to(self, message):
1519 """Copy format-specific state to message insofar as possible."""
1520 if isinstance(message, Message):
1521 return # There's nothing format-specific to explain.
1522 else:
1523 raise TypeError('Cannot convert to specified type')
1524
1525
1526 class ESC[4;38;5;81mMaildirMessage(ESC[4;38;5;149mMessage):
1527 """Message with Maildir-specific properties."""
1528
1529 _type_specific_attributes = ['_subdir', '_info', '_date']
1530
1531 def __init__(self, message=None):
1532 """Initialize a MaildirMessage instance."""
1533 self._subdir = 'new'
1534 self._info = ''
1535 self._date = time.time()
1536 Message.__init__(self, message)
1537
1538 def get_subdir(self):
1539 """Return 'new' or 'cur'."""
1540 return self._subdir
1541
1542 def set_subdir(self, subdir):
1543 """Set subdir to 'new' or 'cur'."""
1544 if subdir == 'new' or subdir == 'cur':
1545 self._subdir = subdir
1546 else:
1547 raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
1548
1549 def get_flags(self):
1550 """Return as a string the flags that are set."""
1551 if self._info.startswith('2,'):
1552 return self._info[2:]
1553 else:
1554 return ''
1555
1556 def set_flags(self, flags):
1557 """Set the given flags and unset all others."""
1558 self._info = '2,' + ''.join(sorted(flags))
1559
1560 def add_flag(self, flag):
1561 """Set the given flag(s) without changing others."""
1562 self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1563
1564 def remove_flag(self, flag):
1565 """Unset the given string flag(s) without changing others."""
1566 if self.get_flags():
1567 self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1568
1569 def get_date(self):
1570 """Return delivery date of message, in seconds since the epoch."""
1571 return self._date
1572
1573 def set_date(self, date):
1574 """Set delivery date of message, in seconds since the epoch."""
1575 try:
1576 self._date = float(date)
1577 except ValueError:
1578 raise TypeError("can't convert to float: %s" % date) from None
1579
1580 def get_info(self):
1581 """Get the message's "info" as a string."""
1582 return self._info
1583
1584 def set_info(self, info):
1585 """Set the message's "info" string."""
1586 if isinstance(info, str):
1587 self._info = info
1588 else:
1589 raise TypeError('info must be a string: %s' % type(info))
1590
1591 def _explain_to(self, message):
1592 """Copy Maildir-specific state to message insofar as possible."""
1593 if isinstance(message, MaildirMessage):
1594 message.set_flags(self.get_flags())
1595 message.set_subdir(self.get_subdir())
1596 message.set_date(self.get_date())
1597 elif isinstance(message, _mboxMMDFMessage):
1598 flags = set(self.get_flags())
1599 if 'S' in flags:
1600 message.add_flag('R')
1601 if self.get_subdir() == 'cur':
1602 message.add_flag('O')
1603 if 'T' in flags:
1604 message.add_flag('D')
1605 if 'F' in flags:
1606 message.add_flag('F')
1607 if 'R' in flags:
1608 message.add_flag('A')
1609 message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
1610 elif isinstance(message, MHMessage):
1611 flags = set(self.get_flags())
1612 if 'S' not in flags:
1613 message.add_sequence('unseen')
1614 if 'R' in flags:
1615 message.add_sequence('replied')
1616 if 'F' in flags:
1617 message.add_sequence('flagged')
1618 elif isinstance(message, BabylMessage):
1619 flags = set(self.get_flags())
1620 if 'S' not in flags:
1621 message.add_label('unseen')
1622 if 'T' in flags:
1623 message.add_label('deleted')
1624 if 'R' in flags:
1625 message.add_label('answered')
1626 if 'P' in flags:
1627 message.add_label('forwarded')
1628 elif isinstance(message, Message):
1629 pass
1630 else:
1631 raise TypeError('Cannot convert to specified type: %s' %
1632 type(message))
1633
1634
1635 class ESC[4;38;5;81m_mboxMMDFMessage(ESC[4;38;5;149mMessage):
1636 """Message with mbox- or MMDF-specific properties."""
1637
1638 _type_specific_attributes = ['_from']
1639
1640 def __init__(self, message=None):
1641 """Initialize an mboxMMDFMessage instance."""
1642 self.set_from('MAILER-DAEMON', True)
1643 if isinstance(message, email.message.Message):
1644 unixfrom = message.get_unixfrom()
1645 if unixfrom is not None and unixfrom.startswith('From '):
1646 self.set_from(unixfrom[5:])
1647 Message.__init__(self, message)
1648
1649 def get_from(self):
1650 """Return contents of "From " line."""
1651 return self._from
1652
1653 def set_from(self, from_, time_=None):
1654 """Set "From " line, formatting and appending time_ if specified."""
1655 if time_ is not None:
1656 if time_ is True:
1657 time_ = time.gmtime()
1658 from_ += ' ' + time.asctime(time_)
1659 self._from = from_
1660
1661 def get_flags(self):
1662 """Return as a string the flags that are set."""
1663 return self.get('Status', '') + self.get('X-Status', '')
1664
1665 def set_flags(self, flags):
1666 """Set the given flags and unset all others."""
1667 flags = set(flags)
1668 status_flags, xstatus_flags = '', ''
1669 for flag in ('R', 'O'):
1670 if flag in flags:
1671 status_flags += flag
1672 flags.remove(flag)
1673 for flag in ('D', 'F', 'A'):
1674 if flag in flags:
1675 xstatus_flags += flag
1676 flags.remove(flag)
1677 xstatus_flags += ''.join(sorted(flags))
1678 try:
1679 self.replace_header('Status', status_flags)
1680 except KeyError:
1681 self.add_header('Status', status_flags)
1682 try:
1683 self.replace_header('X-Status', xstatus_flags)
1684 except KeyError:
1685 self.add_header('X-Status', xstatus_flags)
1686
1687 def add_flag(self, flag):
1688 """Set the given flag(s) without changing others."""
1689 self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1690
1691 def remove_flag(self, flag):
1692 """Unset the given string flag(s) without changing others."""
1693 if 'Status' in self or 'X-Status' in self:
1694 self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1695
1696 def _explain_to(self, message):
1697 """Copy mbox- or MMDF-specific state to message insofar as possible."""
1698 if isinstance(message, MaildirMessage):
1699 flags = set(self.get_flags())
1700 if 'O' in flags:
1701 message.set_subdir('cur')
1702 if 'F' in flags:
1703 message.add_flag('F')
1704 if 'A' in flags:
1705 message.add_flag('R')
1706 if 'R' in flags:
1707 message.add_flag('S')
1708 if 'D' in flags:
1709 message.add_flag('T')
1710 del message['status']
1711 del message['x-status']
1712 maybe_date = ' '.join(self.get_from().split()[-5:])
1713 try:
1714 message.set_date(calendar.timegm(time.strptime(maybe_date,
1715 '%a %b %d %H:%M:%S %Y')))
1716 except (ValueError, OverflowError):
1717 pass
1718 elif isinstance(message, _mboxMMDFMessage):
1719 message.set_flags(self.get_flags())
1720 message.set_from(self.get_from())
1721 elif isinstance(message, MHMessage):
1722 flags = set(self.get_flags())
1723 if 'R' not in flags:
1724 message.add_sequence('unseen')
1725 if 'A' in flags:
1726 message.add_sequence('replied')
1727 if 'F' in flags:
1728 message.add_sequence('flagged')
1729 del message['status']
1730 del message['x-status']
1731 elif isinstance(message, BabylMessage):
1732 flags = set(self.get_flags())
1733 if 'R' not in flags:
1734 message.add_label('unseen')
1735 if 'D' in flags:
1736 message.add_label('deleted')
1737 if 'A' in flags:
1738 message.add_label('answered')
1739 del message['status']
1740 del message['x-status']
1741 elif isinstance(message, Message):
1742 pass
1743 else:
1744 raise TypeError('Cannot convert to specified type: %s' %
1745 type(message))
1746
1747
1748 class ESC[4;38;5;81mmboxMessage(ESC[4;38;5;149m_mboxMMDFMessage):
1749 """Message with mbox-specific properties."""
1750
1751
1752 class ESC[4;38;5;81mMHMessage(ESC[4;38;5;149mMessage):
1753 """Message with MH-specific properties."""
1754
1755 _type_specific_attributes = ['_sequences']
1756
1757 def __init__(self, message=None):
1758 """Initialize an MHMessage instance."""
1759 self._sequences = []
1760 Message.__init__(self, message)
1761
1762 def get_sequences(self):
1763 """Return a list of sequences that include the message."""
1764 return self._sequences[:]
1765
1766 def set_sequences(self, sequences):
1767 """Set the list of sequences that include the message."""
1768 self._sequences = list(sequences)
1769
1770 def add_sequence(self, sequence):
1771 """Add sequence to list of sequences including the message."""
1772 if isinstance(sequence, str):
1773 if not sequence in self._sequences:
1774 self._sequences.append(sequence)
1775 else:
1776 raise TypeError('sequence type must be str: %s' % type(sequence))
1777
1778 def remove_sequence(self, sequence):
1779 """Remove sequence from the list of sequences including the message."""
1780 try:
1781 self._sequences.remove(sequence)
1782 except ValueError:
1783 pass
1784
1785 def _explain_to(self, message):
1786 """Copy MH-specific state to message insofar as possible."""
1787 if isinstance(message, MaildirMessage):
1788 sequences = set(self.get_sequences())
1789 if 'unseen' in sequences:
1790 message.set_subdir('cur')
1791 else:
1792 message.set_subdir('cur')
1793 message.add_flag('S')
1794 if 'flagged' in sequences:
1795 message.add_flag('F')
1796 if 'replied' in sequences:
1797 message.add_flag('R')
1798 elif isinstance(message, _mboxMMDFMessage):
1799 sequences = set(self.get_sequences())
1800 if 'unseen' not in sequences:
1801 message.add_flag('RO')
1802 else:
1803 message.add_flag('O')
1804 if 'flagged' in sequences:
1805 message.add_flag('F')
1806 if 'replied' in sequences:
1807 message.add_flag('A')
1808 elif isinstance(message, MHMessage):
1809 for sequence in self.get_sequences():
1810 message.add_sequence(sequence)
1811 elif isinstance(message, BabylMessage):
1812 sequences = set(self.get_sequences())
1813 if 'unseen' in sequences:
1814 message.add_label('unseen')
1815 if 'replied' in sequences:
1816 message.add_label('answered')
1817 elif isinstance(message, Message):
1818 pass
1819 else:
1820 raise TypeError('Cannot convert to specified type: %s' %
1821 type(message))
1822
1823
1824 class ESC[4;38;5;81mBabylMessage(ESC[4;38;5;149mMessage):
1825 """Message with Babyl-specific properties."""
1826
1827 _type_specific_attributes = ['_labels', '_visible']
1828
1829 def __init__(self, message=None):
1830 """Initialize a BabylMessage instance."""
1831 self._labels = []
1832 self._visible = Message()
1833 Message.__init__(self, message)
1834
1835 def get_labels(self):
1836 """Return a list of labels on the message."""
1837 return self._labels[:]
1838
1839 def set_labels(self, labels):
1840 """Set the list of labels on the message."""
1841 self._labels = list(labels)
1842
1843 def add_label(self, label):
1844 """Add label to list of labels on the message."""
1845 if isinstance(label, str):
1846 if label not in self._labels:
1847 self._labels.append(label)
1848 else:
1849 raise TypeError('label must be a string: %s' % type(label))
1850
1851 def remove_label(self, label):
1852 """Remove label from the list of labels on the message."""
1853 try:
1854 self._labels.remove(label)
1855 except ValueError:
1856 pass
1857
1858 def get_visible(self):
1859 """Return a Message representation of visible headers."""
1860 return Message(self._visible)
1861
1862 def set_visible(self, visible):
1863 """Set the Message representation of visible headers."""
1864 self._visible = Message(visible)
1865
1866 def update_visible(self):
1867 """Update and/or sensibly generate a set of visible headers."""
1868 for header in self._visible.keys():
1869 if header in self:
1870 self._visible.replace_header(header, self[header])
1871 else:
1872 del self._visible[header]
1873 for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
1874 if header in self and header not in self._visible:
1875 self._visible[header] = self[header]
1876
1877 def _explain_to(self, message):
1878 """Copy Babyl-specific state to message insofar as possible."""
1879 if isinstance(message, MaildirMessage):
1880 labels = set(self.get_labels())
1881 if 'unseen' in labels:
1882 message.set_subdir('cur')
1883 else:
1884 message.set_subdir('cur')
1885 message.add_flag('S')
1886 if 'forwarded' in labels or 'resent' in labels:
1887 message.add_flag('P')
1888 if 'answered' in labels:
1889 message.add_flag('R')
1890 if 'deleted' in labels:
1891 message.add_flag('T')
1892 elif isinstance(message, _mboxMMDFMessage):
1893 labels = set(self.get_labels())
1894 if 'unseen' not in labels:
1895 message.add_flag('RO')
1896 else:
1897 message.add_flag('O')
1898 if 'deleted' in labels:
1899 message.add_flag('D')
1900 if 'answered' in labels:
1901 message.add_flag('A')
1902 elif isinstance(message, MHMessage):
1903 labels = set(self.get_labels())
1904 if 'unseen' in labels:
1905 message.add_sequence('unseen')
1906 if 'answered' in labels:
1907 message.add_sequence('replied')
1908 elif isinstance(message, BabylMessage):
1909 message.set_visible(self.get_visible())
1910 for label in self.get_labels():
1911 message.add_label(label)
1912 elif isinstance(message, Message):
1913 pass
1914 else:
1915 raise TypeError('Cannot convert to specified type: %s' %
1916 type(message))
1917
1918
1919 class ESC[4;38;5;81mMMDFMessage(ESC[4;38;5;149m_mboxMMDFMessage):
1920 """Message with MMDF-specific properties."""
1921
1922
1923 class ESC[4;38;5;81m_ProxyFile:
1924 """A read-only wrapper of a file."""
1925
1926 def __init__(self, f, pos=None):
1927 """Initialize a _ProxyFile."""
1928 self._file = f
1929 if pos is None:
1930 self._pos = f.tell()
1931 else:
1932 self._pos = pos
1933
1934 def read(self, size=None):
1935 """Read bytes."""
1936 return self._read(size, self._file.read)
1937
1938 def read1(self, size=None):
1939 """Read bytes."""
1940 return self._read(size, self._file.read1)
1941
1942 def readline(self, size=None):
1943 """Read a line."""
1944 return self._read(size, self._file.readline)
1945
1946 def readlines(self, sizehint=None):
1947 """Read multiple lines."""
1948 result = []
1949 for line in self:
1950 result.append(line)
1951 if sizehint is not None:
1952 sizehint -= len(line)
1953 if sizehint <= 0:
1954 break
1955 return result
1956
1957 def __iter__(self):
1958 """Iterate over lines."""
1959 while True:
1960 line = self.readline()
1961 if not line:
1962 return
1963 yield line
1964
1965 def tell(self):
1966 """Return the position."""
1967 return self._pos
1968
1969 def seek(self, offset, whence=0):
1970 """Change position."""
1971 if whence == 1:
1972 self._file.seek(self._pos)
1973 self._file.seek(offset, whence)
1974 self._pos = self._file.tell()
1975
1976 def close(self):
1977 """Close the file."""
1978 if hasattr(self, '_file'):
1979 try:
1980 if hasattr(self._file, 'close'):
1981 self._file.close()
1982 finally:
1983 del self._file
1984
1985 def _read(self, size, read_method):
1986 """Read size bytes using read_method."""
1987 if size is None:
1988 size = -1
1989 self._file.seek(self._pos)
1990 result = read_method(size)
1991 self._pos = self._file.tell()
1992 return result
1993
1994 def __enter__(self):
1995 """Context management protocol support."""
1996 return self
1997
1998 def __exit__(self, *exc):
1999 self.close()
2000
2001 def readable(self):
2002 return self._file.readable()
2003
2004 def writable(self):
2005 return self._file.writable()
2006
2007 def seekable(self):
2008 return self._file.seekable()
2009
2010 def flush(self):
2011 return self._file.flush()
2012
2013 @property
2014 def closed(self):
2015 if not hasattr(self, '_file'):
2016 return True
2017 if not hasattr(self._file, 'closed'):
2018 return False
2019 return self._file.closed
2020
2021 __class_getitem__ = classmethod(GenericAlias)
2022
2023
2024 class ESC[4;38;5;81m_PartialFile(ESC[4;38;5;149m_ProxyFile):
2025 """A read-only wrapper of part of a file."""
2026
2027 def __init__(self, f, start=None, stop=None):
2028 """Initialize a _PartialFile."""
2029 _ProxyFile.__init__(self, f, start)
2030 self._start = start
2031 self._stop = stop
2032
2033 def tell(self):
2034 """Return the position with respect to start."""
2035 return _ProxyFile.tell(self) - self._start
2036
2037 def seek(self, offset, whence=0):
2038 """Change position, possibly with respect to start or stop."""
2039 if whence == 0:
2040 self._pos = self._start
2041 whence = 1
2042 elif whence == 2:
2043 self._pos = self._stop
2044 whence = 1
2045 _ProxyFile.seek(self, offset, whence)
2046
2047 def _read(self, size, read_method):
2048 """Read size bytes using read_method, honoring start and stop."""
2049 remaining = self._stop - self._pos
2050 if remaining <= 0:
2051 return b''
2052 if size is None or size < 0 or size > remaining:
2053 size = remaining
2054 return _ProxyFile._read(self, size, read_method)
2055
2056 def close(self):
2057 # do *not* close the underlying file object for partial files,
2058 # since it's global to the mailbox object
2059 if hasattr(self, '_file'):
2060 del self._file
2061
2062
2063 def _lock_file(f, dotlock=True):
2064 """Lock file f using lockf and dot locking."""
2065 dotlock_done = False
2066 try:
2067 if fcntl:
2068 try:
2069 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
2070 except OSError as e:
2071 if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
2072 raise ExternalClashError('lockf: lock unavailable: %s' %
2073 f.name)
2074 else:
2075 raise
2076 if dotlock:
2077 try:
2078 pre_lock = _create_temporary(f.name + '.lock')
2079 pre_lock.close()
2080 except OSError as e:
2081 if e.errno in (errno.EACCES, errno.EROFS):
2082 return # Without write access, just skip dotlocking.
2083 else:
2084 raise
2085 try:
2086 try:
2087 os.link(pre_lock.name, f.name + '.lock')
2088 dotlock_done = True
2089 except (AttributeError, PermissionError):
2090 os.rename(pre_lock.name, f.name + '.lock')
2091 dotlock_done = True
2092 else:
2093 os.unlink(pre_lock.name)
2094 except FileExistsError:
2095 os.remove(pre_lock.name)
2096 raise ExternalClashError('dot lock unavailable: %s' %
2097 f.name)
2098 except:
2099 if fcntl:
2100 fcntl.lockf(f, fcntl.LOCK_UN)
2101 if dotlock_done:
2102 os.remove(f.name + '.lock')
2103 raise
2104
2105 def _unlock_file(f):
2106 """Unlock file f using lockf and dot locking."""
2107 if fcntl:
2108 fcntl.lockf(f, fcntl.LOCK_UN)
2109 if os.path.exists(f.name + '.lock'):
2110 os.remove(f.name + '.lock')
2111
2112 def _create_carefully(path):
2113 """Create a file if it doesn't exist and open for reading and writing."""
2114 fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666)
2115 try:
2116 return open(path, 'rb+')
2117 finally:
2118 os.close(fd)
2119
2120 def _create_temporary(path):
2121 """Create a temp file based on path and open for reading and writing."""
2122 return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
2123 socket.gethostname(),
2124 os.getpid()))
2125
2126 def _sync_flush(f):
2127 """Ensure changes to file f are physically on disk."""
2128 f.flush()
2129 if hasattr(os, 'fsync'):
2130 os.fsync(f.fileno())
2131
2132 def _sync_close(f):
2133 """Close file f, ensuring all changes are physically on disk."""
2134 _sync_flush(f)
2135 f.close()
2136
2137
2138 class ESC[4;38;5;81mError(ESC[4;38;5;149mException):
2139 """Raised for module-specific errors."""
2140
2141 class ESC[4;38;5;81mNoSuchMailboxError(ESC[4;38;5;149mError):
2142 """The specified mailbox does not exist and won't be created."""
2143
2144 class ESC[4;38;5;81mNotEmptyError(ESC[4;38;5;149mError):
2145 """The specified mailbox is not empty and deletion was requested."""
2146
2147 class ESC[4;38;5;81mExternalClashError(ESC[4;38;5;149mError):
2148 """Another process caused an action to fail."""
2149
2150 class ESC[4;38;5;81mFormatError(ESC[4;38;5;149mError):
2151 """A file appears to have an invalid format."""