1 """Test the binascii C module."""
2
3 import unittest
4 import binascii
5 import array
6 import re
7 from test.support import bigmemtest, _1G, _4G, warnings_helper
8
9
10 # Note: "*_hex" functions are aliases for "(un)hexlify"
11 b2a_functions = ['b2a_base64', 'b2a_hex', 'b2a_qp', 'b2a_uu',
12 'hexlify']
13 a2b_functions = ['a2b_base64', 'a2b_hex', 'a2b_qp', 'a2b_uu',
14 'unhexlify']
15 all_functions = a2b_functions + b2a_functions + ['crc32', 'crc_hqx']
16
17
18 class ESC[4;38;5;81mBinASCIITest(ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
19
20 type2test = bytes
21 # Create binary test data
22 rawdata = b"The quick brown fox jumps over the lazy dog.\r\n"
23 # Be slow so we don't depend on other modules
24 rawdata += bytes(range(256))
25 rawdata += b"\r\nHello world.\n"
26
27 def setUp(self):
28 self.data = self.type2test(self.rawdata)
29
30 def test_exceptions(self):
31 # Check module exceptions
32 self.assertTrue(issubclass(binascii.Error, Exception))
33 self.assertTrue(issubclass(binascii.Incomplete, Exception))
34
35 def test_functions(self):
36 # Check presence of all functions
37 for name in all_functions:
38 self.assertTrue(hasattr(getattr(binascii, name), '__call__'))
39 self.assertRaises(TypeError, getattr(binascii, name))
40
41 def test_returned_value(self):
42 # Limit to the minimum of all limits (b2a_uu)
43 MAX_ALL = 45
44 raw = self.rawdata[:MAX_ALL]
45 for fa, fb in zip(a2b_functions, b2a_functions):
46 a2b = getattr(binascii, fa)
47 b2a = getattr(binascii, fb)
48 try:
49 a = b2a(self.type2test(raw))
50 res = a2b(self.type2test(a))
51 except Exception as err:
52 self.fail("{}/{} conversion raises {!r}".format(fb, fa, err))
53 self.assertEqual(res, raw, "{}/{} conversion: "
54 "{!r} != {!r}".format(fb, fa, res, raw))
55 self.assertIsInstance(res, bytes)
56 self.assertIsInstance(a, bytes)
57 self.assertLess(max(a), 128)
58 self.assertIsInstance(binascii.crc_hqx(raw, 0), int)
59 self.assertIsInstance(binascii.crc32(raw), int)
60
61 def test_base64valid(self):
62 # Test base64 with valid data
63 MAX_BASE64 = 57
64 lines = []
65 for i in range(0, len(self.rawdata), MAX_BASE64):
66 b = self.type2test(self.rawdata[i:i+MAX_BASE64])
67 a = binascii.b2a_base64(b)
68 lines.append(a)
69 res = bytes()
70 for line in lines:
71 a = self.type2test(line)
72 b = binascii.a2b_base64(a)
73 res += b
74 self.assertEqual(res, self.rawdata)
75
76 def test_base64invalid(self):
77 # Test base64 with random invalid characters sprinkled throughout
78 # (This requires a new version of binascii.)
79 MAX_BASE64 = 57
80 lines = []
81 for i in range(0, len(self.data), MAX_BASE64):
82 b = self.type2test(self.rawdata[i:i+MAX_BASE64])
83 a = binascii.b2a_base64(b)
84 lines.append(a)
85
86 fillers = bytearray()
87 valid = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/"
88 for i in range(256):
89 if i not in valid:
90 fillers.append(i)
91 def addnoise(line):
92 noise = fillers
93 ratio = len(line) // len(noise)
94 res = bytearray()
95 while line and noise:
96 if len(line) // len(noise) > ratio:
97 c, line = line[0], line[1:]
98 else:
99 c, noise = noise[0], noise[1:]
100 res.append(c)
101 return res + noise + line
102 res = bytearray()
103 for line in map(addnoise, lines):
104 a = self.type2test(line)
105 b = binascii.a2b_base64(a)
106 res += b
107 self.assertEqual(res, self.rawdata)
108
109 # Test base64 with just invalid characters, which should return
110 # empty strings. TBD: shouldn't it raise an exception instead ?
111 self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'')
112
113 def test_base64_strict_mode(self):
114 # Test base64 with strict mode on
115 def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes):
116 with self.assertRaisesRegex(binascii.Error, assert_regex):
117 binascii.a2b_base64(self.type2test(data), strict_mode=True)
118 self.assertEqual(binascii.a2b_base64(self.type2test(data), strict_mode=False),
119 non_strict_mode_expected_result)
120 self.assertEqual(binascii.a2b_base64(self.type2test(data)),
121 non_strict_mode_expected_result)
122
123 def assertExcessData(data, non_strict_mode_expected_result: bytes):
124 _assertRegexTemplate(r'(?i)Excess data', data, non_strict_mode_expected_result)
125
126 def assertNonBase64Data(data, non_strict_mode_expected_result: bytes):
127 _assertRegexTemplate(r'(?i)Only base64 data', data, non_strict_mode_expected_result)
128
129 def assertLeadingPadding(data, non_strict_mode_expected_result: bytes):
130 _assertRegexTemplate(r'(?i)Leading padding', data, non_strict_mode_expected_result)
131
132 def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes):
133 _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result)
134
135 # Test excess data exceptions
136 assertExcessData(b'ab==a', b'i')
137 assertExcessData(b'ab===', b'i')
138 assertExcessData(b'ab==:', b'i')
139 assertExcessData(b'abc=a', b'i\xb7')
140 assertExcessData(b'abc=:', b'i\xb7')
141 assertExcessData(b'ab==\n', b'i')
142
143 # Test non-base64 data exceptions
144 assertNonBase64Data(b'\nab==', b'i')
145 assertNonBase64Data(b'ab:(){:|:&};:==', b'i')
146 assertNonBase64Data(b'a\nb==', b'i')
147 assertNonBase64Data(b'a\x00b==', b'i')
148
149 # Test malformed padding
150 assertLeadingPadding(b'=', b'')
151 assertLeadingPadding(b'==', b'')
152 assertLeadingPadding(b'===', b'')
153 assertDiscontinuousPadding(b'ab=c=', b'i\xb7')
154 assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b')
155
156
157 def test_base64errors(self):
158 # Test base64 with invalid padding
159 def assertIncorrectPadding(data):
160 with self.assertRaisesRegex(binascii.Error, r'(?i)Incorrect padding'):
161 binascii.a2b_base64(self.type2test(data))
162
163 assertIncorrectPadding(b'ab')
164 assertIncorrectPadding(b'ab=')
165 assertIncorrectPadding(b'abc')
166 assertIncorrectPadding(b'abcdef')
167 assertIncorrectPadding(b'abcdef=')
168 assertIncorrectPadding(b'abcdefg')
169 assertIncorrectPadding(b'a=b=')
170 assertIncorrectPadding(b'a\nb=')
171
172 # Test base64 with invalid number of valid characters (1 mod 4)
173 def assertInvalidLength(data):
174 n_data_chars = len(re.sub(br'[^A-Za-z0-9/+]', br'', data))
175 expected_errmsg_re = \
176 r'(?i)Invalid.+number of data characters.+' + str(n_data_chars)
177 with self.assertRaisesRegex(binascii.Error, expected_errmsg_re):
178 binascii.a2b_base64(self.type2test(data))
179
180 assertInvalidLength(b'a')
181 assertInvalidLength(b'a=')
182 assertInvalidLength(b'a==')
183 assertInvalidLength(b'a===')
184 assertInvalidLength(b'a' * 5)
185 assertInvalidLength(b'a' * (4 * 87 + 1))
186 assertInvalidLength(b'A\tB\nC ??DE') # only 5 valid characters
187
188 def test_uu(self):
189 MAX_UU = 45
190 for backtick in (True, False):
191 lines = []
192 for i in range(0, len(self.data), MAX_UU):
193 b = self.type2test(self.rawdata[i:i+MAX_UU])
194 a = binascii.b2a_uu(b, backtick=backtick)
195 lines.append(a)
196 res = bytes()
197 for line in lines:
198 a = self.type2test(line)
199 b = binascii.a2b_uu(a)
200 res += b
201 self.assertEqual(res, self.rawdata)
202
203 self.assertEqual(binascii.a2b_uu(b"\x7f"), b"\x00"*31)
204 self.assertEqual(binascii.a2b_uu(b"\x80"), b"\x00"*32)
205 self.assertEqual(binascii.a2b_uu(b"\xff"), b"\x00"*31)
206 self.assertRaises(binascii.Error, binascii.a2b_uu, b"\xff\x00")
207 self.assertRaises(binascii.Error, binascii.a2b_uu, b"!!!!")
208 self.assertRaises(binascii.Error, binascii.b2a_uu, 46*b"!")
209
210 # Issue #7701 (crash on a pydebug build)
211 self.assertEqual(binascii.b2a_uu(b'x'), b'!> \n')
212
213 self.assertEqual(binascii.b2a_uu(b''), b' \n')
214 self.assertEqual(binascii.b2a_uu(b'', backtick=True), b'`\n')
215 self.assertEqual(binascii.a2b_uu(b' \n'), b'')
216 self.assertEqual(binascii.a2b_uu(b'`\n'), b'')
217 self.assertEqual(binascii.b2a_uu(b'\x00Cat'), b'$ $-A= \n')
218 self.assertEqual(binascii.b2a_uu(b'\x00Cat', backtick=True),
219 b'$`$-A=```\n')
220 self.assertEqual(binascii.a2b_uu(b'$`$-A=```\n'),
221 binascii.a2b_uu(b'$ $-A= \n'))
222 with self.assertRaises(TypeError):
223 binascii.b2a_uu(b"", True)
224
225 def test_crc_hqx(self):
226 crc = binascii.crc_hqx(self.type2test(b"Test the CRC-32 of"), 0)
227 crc = binascii.crc_hqx(self.type2test(b" this string."), crc)
228 self.assertEqual(crc, 14290)
229
230 self.assertRaises(TypeError, binascii.crc_hqx)
231 self.assertRaises(TypeError, binascii.crc_hqx, self.type2test(b''))
232
233 for crc in 0, 1, 0x1234, 0x12345, 0x12345678, -1:
234 self.assertEqual(binascii.crc_hqx(self.type2test(b''), crc),
235 crc & 0xffff)
236
237 def test_crc32(self):
238 crc = binascii.crc32(self.type2test(b"Test the CRC-32 of"))
239 crc = binascii.crc32(self.type2test(b" this string."), crc)
240 self.assertEqual(crc, 1571220330)
241
242 self.assertRaises(TypeError, binascii.crc32)
243
244 def test_hex(self):
245 # test hexlification
246 s = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000'
247 t = binascii.b2a_hex(self.type2test(s))
248 u = binascii.a2b_hex(self.type2test(t))
249 self.assertEqual(s, u)
250 self.assertRaises(binascii.Error, binascii.a2b_hex, t[:-1])
251 self.assertRaises(binascii.Error, binascii.a2b_hex, t[:-1] + b'q')
252 self.assertRaises(binascii.Error, binascii.a2b_hex, bytes([255, 255]))
253 self.assertRaises(binascii.Error, binascii.a2b_hex, b'0G')
254 self.assertRaises(binascii.Error, binascii.a2b_hex, b'0g')
255 self.assertRaises(binascii.Error, binascii.a2b_hex, b'G0')
256 self.assertRaises(binascii.Error, binascii.a2b_hex, b'g0')
257
258 # Confirm that b2a_hex == hexlify and a2b_hex == unhexlify
259 self.assertEqual(binascii.hexlify(self.type2test(s)), t)
260 self.assertEqual(binascii.unhexlify(self.type2test(t)), u)
261
262 def test_hex_separator(self):
263 """Test that hexlify and b2a_hex are binary versions of bytes.hex."""
264 # Logic of separators is tested in test_bytes.py. This checks that
265 # arg parsing works and exercises the direct to bytes object code
266 # path within pystrhex.c.
267 s = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000'
268 self.assertEqual(binascii.hexlify(self.type2test(s)), s.hex().encode('ascii'))
269 expected8 = s.hex('.', 8).encode('ascii')
270 self.assertEqual(binascii.hexlify(self.type2test(s), '.', 8), expected8)
271 expected1 = s.hex(':').encode('ascii')
272 self.assertEqual(binascii.b2a_hex(self.type2test(s), ':'), expected1)
273
274 def test_qp(self):
275 type2test = self.type2test
276 a2b_qp = binascii.a2b_qp
277 b2a_qp = binascii.b2a_qp
278
279 a2b_qp(data=b"", header=False) # Keyword arguments allowed
280
281 # A test for SF bug 534347 (segfaults without the proper fix)
282 try:
283 a2b_qp(b"", **{1:1})
284 except TypeError:
285 pass
286 else:
287 self.fail("binascii.a2b_qp(**{1:1}) didn't raise TypeError")
288
289 self.assertEqual(a2b_qp(type2test(b"=")), b"")
290 self.assertEqual(a2b_qp(type2test(b"= ")), b"= ")
291 self.assertEqual(a2b_qp(type2test(b"==")), b"=")
292 self.assertEqual(a2b_qp(type2test(b"=\nAB")), b"AB")
293 self.assertEqual(a2b_qp(type2test(b"=\r\nAB")), b"AB")
294 self.assertEqual(a2b_qp(type2test(b"=\rAB")), b"") # ?
295 self.assertEqual(a2b_qp(type2test(b"=\rAB\nCD")), b"CD") # ?
296 self.assertEqual(a2b_qp(type2test(b"=AB")), b"\xab")
297 self.assertEqual(a2b_qp(type2test(b"=ab")), b"\xab")
298 self.assertEqual(a2b_qp(type2test(b"=AX")), b"=AX")
299 self.assertEqual(a2b_qp(type2test(b"=XA")), b"=XA")
300 self.assertEqual(a2b_qp(type2test(b"=AB")[:-1]), b"=A")
301
302 self.assertEqual(a2b_qp(type2test(b'_')), b'_')
303 self.assertEqual(a2b_qp(type2test(b'_'), header=True), b' ')
304
305 self.assertRaises(TypeError, b2a_qp, foo="bar")
306 self.assertEqual(a2b_qp(type2test(b"=00\r\n=00")), b"\x00\r\n\x00")
307 self.assertEqual(b2a_qp(type2test(b"\xff\r\n\xff\n\xff")),
308 b"=FF\r\n=FF\r\n=FF")
309 self.assertEqual(b2a_qp(type2test(b"0"*75+b"\xff\r\n\xff\r\n\xff")),
310 b"0"*75+b"=\r\n=FF\r\n=FF\r\n=FF")
311
312 self.assertEqual(b2a_qp(type2test(b'\x7f')), b'=7F')
313 self.assertEqual(b2a_qp(type2test(b'=')), b'=3D')
314
315 self.assertEqual(b2a_qp(type2test(b'_')), b'_')
316 self.assertEqual(b2a_qp(type2test(b'_'), header=True), b'=5F')
317 self.assertEqual(b2a_qp(type2test(b'x y'), header=True), b'x_y')
318 self.assertEqual(b2a_qp(type2test(b'x '), header=True), b'x=20')
319 self.assertEqual(b2a_qp(type2test(b'x y'), header=True, quotetabs=True),
320 b'x=20y')
321 self.assertEqual(b2a_qp(type2test(b'x\ty'), header=True), b'x\ty')
322
323 self.assertEqual(b2a_qp(type2test(b' ')), b'=20')
324 self.assertEqual(b2a_qp(type2test(b'\t')), b'=09')
325 self.assertEqual(b2a_qp(type2test(b' x')), b' x')
326 self.assertEqual(b2a_qp(type2test(b'\tx')), b'\tx')
327 self.assertEqual(b2a_qp(type2test(b' x')[:-1]), b'=20')
328 self.assertEqual(b2a_qp(type2test(b'\tx')[:-1]), b'=09')
329 self.assertEqual(b2a_qp(type2test(b'\0')), b'=00')
330
331 self.assertEqual(b2a_qp(type2test(b'\0\n')), b'=00\n')
332 self.assertEqual(b2a_qp(type2test(b'\0\n'), quotetabs=True), b'=00\n')
333
334 self.assertEqual(b2a_qp(type2test(b'x y\tz')), b'x y\tz')
335 self.assertEqual(b2a_qp(type2test(b'x y\tz'), quotetabs=True),
336 b'x=20y=09z')
337 self.assertEqual(b2a_qp(type2test(b'x y\tz'), istext=False),
338 b'x y\tz')
339 self.assertEqual(b2a_qp(type2test(b'x \ny\t\n')),
340 b'x=20\ny=09\n')
341 self.assertEqual(b2a_qp(type2test(b'x \ny\t\n'), quotetabs=True),
342 b'x=20\ny=09\n')
343 self.assertEqual(b2a_qp(type2test(b'x \ny\t\n'), istext=False),
344 b'x =0Ay\t=0A')
345 self.assertEqual(b2a_qp(type2test(b'x \ry\t\r')),
346 b'x \ry\t\r')
347 self.assertEqual(b2a_qp(type2test(b'x \ry\t\r'), quotetabs=True),
348 b'x=20\ry=09\r')
349 self.assertEqual(b2a_qp(type2test(b'x \ry\t\r'), istext=False),
350 b'x =0Dy\t=0D')
351 self.assertEqual(b2a_qp(type2test(b'x \r\ny\t\r\n')),
352 b'x=20\r\ny=09\r\n')
353 self.assertEqual(b2a_qp(type2test(b'x \r\ny\t\r\n'), quotetabs=True),
354 b'x=20\r\ny=09\r\n')
355 self.assertEqual(b2a_qp(type2test(b'x \r\ny\t\r\n'), istext=False),
356 b'x =0D=0Ay\t=0D=0A')
357
358 self.assertEqual(b2a_qp(type2test(b'x \r\n')[:-1]), b'x \r')
359 self.assertEqual(b2a_qp(type2test(b'x\t\r\n')[:-1]), b'x\t\r')
360 self.assertEqual(b2a_qp(type2test(b'x \r\n')[:-1], quotetabs=True),
361 b'x=20\r')
362 self.assertEqual(b2a_qp(type2test(b'x\t\r\n')[:-1], quotetabs=True),
363 b'x=09\r')
364 self.assertEqual(b2a_qp(type2test(b'x \r\n')[:-1], istext=False),
365 b'x =0D')
366 self.assertEqual(b2a_qp(type2test(b'x\t\r\n')[:-1], istext=False),
367 b'x\t=0D')
368
369 self.assertEqual(b2a_qp(type2test(b'.')), b'=2E')
370 self.assertEqual(b2a_qp(type2test(b'.\n')), b'=2E\n')
371 self.assertEqual(b2a_qp(type2test(b'.\r')), b'=2E\r')
372 self.assertEqual(b2a_qp(type2test(b'.\0')), b'=2E=00')
373 self.assertEqual(b2a_qp(type2test(b'a.\n')), b'a.\n')
374 self.assertEqual(b2a_qp(type2test(b'.a')[:-1]), b'=2E')
375
376 def test_empty_string(self):
377 # A test for SF bug #1022953. Make sure SystemError is not raised.
378 empty = self.type2test(b'')
379 for func in all_functions:
380 if func == 'crc_hqx':
381 # crc_hqx needs 2 arguments
382 binascii.crc_hqx(empty, 0)
383 continue
384 f = getattr(binascii, func)
385 try:
386 f(empty)
387 except Exception as err:
388 self.fail("{}({!r}) raises {!r}".format(func, empty, err))
389
390 def test_unicode_b2a(self):
391 # Unicode strings are not accepted by b2a_* functions.
392 for func in set(all_functions) - set(a2b_functions):
393 try:
394 self.assertRaises(TypeError, getattr(binascii, func), "test")
395 except Exception as err:
396 self.fail('{}("test") raises {!r}'.format(func, err))
397 # crc_hqx needs 2 arguments
398 self.assertRaises(TypeError, binascii.crc_hqx, "test", 0)
399
400 def test_unicode_a2b(self):
401 # Unicode strings are accepted by a2b_* functions.
402 MAX_ALL = 45
403 raw = self.rawdata[:MAX_ALL]
404 for fa, fb in zip(a2b_functions, b2a_functions):
405 a2b = getattr(binascii, fa)
406 b2a = getattr(binascii, fb)
407 try:
408 a = b2a(self.type2test(raw))
409 binary_res = a2b(a)
410 a = a.decode('ascii')
411 res = a2b(a)
412 except Exception as err:
413 self.fail("{}/{} conversion raises {!r}".format(fb, fa, err))
414 self.assertEqual(res, raw, "{}/{} conversion: "
415 "{!r} != {!r}".format(fb, fa, res, raw))
416 self.assertEqual(res, binary_res)
417 self.assertIsInstance(res, bytes)
418 # non-ASCII string
419 self.assertRaises(ValueError, a2b, "\x80")
420
421 def test_b2a_base64_newline(self):
422 # Issue #25357: test newline parameter
423 b = self.type2test(b'hello')
424 self.assertEqual(binascii.b2a_base64(b),
425 b'aGVsbG8=\n')
426 self.assertEqual(binascii.b2a_base64(b, newline=True),
427 b'aGVsbG8=\n')
428 self.assertEqual(binascii.b2a_base64(b, newline=False),
429 b'aGVsbG8=')
430
431 def test_c_contiguity(self):
432 m = memoryview(bytearray(b'noncontig'))
433 noncontig_writable = m[::-2]
434 with self.assertRaises(BufferError):
435 binascii.b2a_hex(noncontig_writable)
436
437
438 class ESC[4;38;5;81mArrayBinASCIITest(ESC[4;38;5;149mBinASCIITest):
439 def type2test(self, s):
440 return array.array('B', list(s))
441
442
443 class ESC[4;38;5;81mBytearrayBinASCIITest(ESC[4;38;5;149mBinASCIITest):
444 type2test = bytearray
445
446
447 class ESC[4;38;5;81mMemoryviewBinASCIITest(ESC[4;38;5;149mBinASCIITest):
448 type2test = memoryview
449
450 class ESC[4;38;5;81mChecksumBigBufferTestCase(ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
451 """bpo-38256 - check that inputs >=4 GiB are handled correctly."""
452
453 @bigmemtest(size=_4G + 4, memuse=1, dry_run=False)
454 def test_big_buffer(self, size):
455 data = b"nyan" * (_1G + 1)
456 self.assertEqual(binascii.crc32(data), 1044521549)
457
458
459 if __name__ == "__main__":
460 unittest.main()