1 """ Routines for manipulating RFC2047 encoded words.
2
3 This is currently a package-private API, but will be considered for promotion
4 to a public API if there is demand.
5
6 """
7
8 # An ecoded word looks like this:
9 #
10 # =?charset[*lang]?cte?encoded_string?=
11 #
12 # for more information about charset see the charset module. Here it is one
13 # of the preferred MIME charset names (hopefully; you never know when parsing).
14 # cte (Content Transfer Encoding) is either 'q' or 'b' (ignoring case). In
15 # theory other letters could be used for other encodings, but in practice this
16 # (almost?) never happens. There could be a public API for adding entries
17 # to the CTE tables, but YAGNI for now. 'q' is Quoted Printable, 'b' is
18 # Base64. The meaning of encoded_string should be obvious. 'lang' is optional
19 # as indicated by the brackets (they are not part of the syntax) but is almost
20 # never encountered in practice.
21 #
22 # The general interface for a CTE decoder is that it takes the encoded_string
23 # as its argument, and returns a tuple (cte_decoded_string, defects). The
24 # cte_decoded_string is the original binary that was encoded using the
25 # specified cte. 'defects' is a list of MessageDefect instances indicating any
26 # problems encountered during conversion. 'charset' and 'lang' are the
27 # corresponding strings extracted from the EW, case preserved.
28 #
29 # The general interface for a CTE encoder is that it takes a binary sequence
30 # as input and returns the cte_encoded_string, which is an ascii-only string.
31 #
32 # Each decoder must also supply a length function that takes the binary
33 # sequence as its argument and returns the length of the resulting encoded
34 # string.
35 #
36 # The main API functions for the module are decode, which calls the decoder
37 # referenced by the cte specifier, and encode, which adds the appropriate
38 # RFC 2047 "chrome" to the encoded string, and can optionally automatically
39 # select the shortest possible encoding. See their docstrings below for
40 # details.
41
42 import re
43 import base64
44 import binascii
45 import functools
46 from string import ascii_letters, digits
47 from email import errors
48
49 __all__ = ['decode_q',
50 'encode_q',
51 'decode_b',
52 'encode_b',
53 'len_q',
54 'len_b',
55 'decode',
56 'encode',
57 ]
58
59 #
60 # Quoted Printable
61 #
62
63 # regex based decoder.
64 _q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub,
65 lambda m: bytes.fromhex(m.group(1).decode()))
66
67 def decode_q(encoded):
68 encoded = encoded.replace(b'_', b' ')
69 return _q_byte_subber(encoded), []
70
71
72 # dict mapping bytes to their encoded form
73 class ESC[4;38;5;81m_QByteMap(ESC[4;38;5;149mdict):
74
75 safe = b'-!*+/' + ascii_letters.encode('ascii') + digits.encode('ascii')
76
77 def __missing__(self, key):
78 if key in self.safe:
79 self[key] = chr(key)
80 else:
81 self[key] = "={:02X}".format(key)
82 return self[key]
83
84 _q_byte_map = _QByteMap()
85
86 # In headers spaces are mapped to '_'.
87 _q_byte_map[ord(' ')] = '_'
88
89 def encode_q(bstring):
90 return ''.join(_q_byte_map[x] for x in bstring)
91
92 def len_q(bstring):
93 return sum(len(_q_byte_map[x]) for x in bstring)
94
95
96 #
97 # Base64
98 #
99
100 def decode_b(encoded):
101 # First try encoding with validate=True, fixing the padding if needed.
102 # This will succeed only if encoded includes no invalid characters.
103 pad_err = len(encoded) % 4
104 missing_padding = b'==='[:4-pad_err] if pad_err else b''
105 try:
106 return (
107 base64.b64decode(encoded + missing_padding, validate=True),
108 [errors.InvalidBase64PaddingDefect()] if pad_err else [],
109 )
110 except binascii.Error:
111 # Since we had correct padding, this is likely an invalid char error.
112 #
113 # The non-alphabet characters are ignored as far as padding
114 # goes, but we don't know how many there are. So try without adding
115 # padding to see if it works.
116 try:
117 return (
118 base64.b64decode(encoded, validate=False),
119 [errors.InvalidBase64CharactersDefect()],
120 )
121 except binascii.Error:
122 # Add as much padding as could possibly be necessary (extra padding
123 # is ignored).
124 try:
125 return (
126 base64.b64decode(encoded + b'==', validate=False),
127 [errors.InvalidBase64CharactersDefect(),
128 errors.InvalidBase64PaddingDefect()],
129 )
130 except binascii.Error:
131 # This only happens when the encoded string's length is 1 more
132 # than a multiple of 4, which is invalid.
133 #
134 # bpo-27397: Just return the encoded string since there's no
135 # way to decode.
136 return encoded, [errors.InvalidBase64LengthDefect()]
137
138 def encode_b(bstring):
139 return base64.b64encode(bstring).decode('ascii')
140
141 def len_b(bstring):
142 groups_of_3, leftover = divmod(len(bstring), 3)
143 # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in.
144 return groups_of_3 * 4 + (4 if leftover else 0)
145
146
147 _cte_decoders = {
148 'q': decode_q,
149 'b': decode_b,
150 }
151
152 def decode(ew):
153 """Decode encoded word and return (string, charset, lang, defects) tuple.
154
155 An RFC 2047/2243 encoded word has the form:
156
157 =?charset*lang?cte?encoded_string?=
158
159 where '*lang' may be omitted but the other parts may not be.
160
161 This function expects exactly such a string (that is, it does not check the
162 syntax and may raise errors if the string is not well formed), and returns
163 the encoded_string decoded first from its Content Transfer Encoding and
164 then from the resulting bytes into unicode using the specified charset. If
165 the cte-decoded string does not successfully decode using the specified
166 character set, a defect is added to the defects list and the unknown octets
167 are replaced by the unicode 'unknown' character \\uFDFF.
168
169 The specified charset and language are returned. The default for language,
170 which is rarely if ever encountered, is the empty string.
171
172 """
173 _, charset, cte, cte_string, _ = ew.split('?')
174 charset, _, lang = charset.partition('*')
175 cte = cte.lower()
176 # Recover the original bytes and do CTE decoding.
177 bstring = cte_string.encode('ascii', 'surrogateescape')
178 bstring, defects = _cte_decoders[cte](bstring)
179 # Turn the CTE decoded bytes into unicode.
180 try:
181 string = bstring.decode(charset)
182 except UnicodeDecodeError:
183 defects.append(errors.UndecodableBytesDefect("Encoded word "
184 f"contains bytes not decodable using {charset!r} charset"))
185 string = bstring.decode(charset, 'surrogateescape')
186 except (LookupError, UnicodeEncodeError):
187 string = bstring.decode('ascii', 'surrogateescape')
188 if charset.lower() != 'unknown-8bit':
189 defects.append(errors.CharsetError(f"Unknown charset {charset!r} "
190 f"in encoded word; decoded as unknown bytes"))
191 return string, charset, lang, defects
192
193
194 _cte_encoders = {
195 'q': encode_q,
196 'b': encode_b,
197 }
198
199 _cte_encode_length = {
200 'q': len_q,
201 'b': len_b,
202 }
203
204 def encode(string, charset='utf-8', encoding=None, lang=''):
205 """Encode string using the CTE encoding that produces the shorter result.
206
207 Produces an RFC 2047/2243 encoded word of the form:
208
209 =?charset*lang?cte?encoded_string?=
210
211 where '*lang' is omitted unless the 'lang' parameter is given a value.
212 Optional argument charset (defaults to utf-8) specifies the charset to use
213 to encode the string to binary before CTE encoding it. Optional argument
214 'encoding' is the cte specifier for the encoding that should be used ('q'
215 or 'b'); if it is None (the default) the encoding which produces the
216 shortest encoded sequence is used, except that 'q' is preferred if it is up
217 to five characters longer. Optional argument 'lang' (default '') gives the
218 RFC 2243 language string to specify in the encoded word.
219
220 """
221 if charset == 'unknown-8bit':
222 bstring = string.encode('ascii', 'surrogateescape')
223 else:
224 bstring = string.encode(charset)
225 if encoding is None:
226 qlen = _cte_encode_length['q'](bstring)
227 blen = _cte_encode_length['b'](bstring)
228 # Bias toward q. 5 is arbitrary.
229 encoding = 'q' if qlen - blen < 5 else 'b'
230 encoded = _cte_encoders[encoding](bstring)
231 if lang:
232 lang = '*' + lang
233 return "=?{}{}?{}?{}?=".format(charset, lang, encoding, encoded)