python (3.11.7)
1 """Fallback pure Python implementation of msgpack"""
2 from datetime import datetime as _DateTime
3 import sys
4 import struct
5
6
7 PY2 = sys.version_info[0] == 2
8 if PY2:
9 int_types = (int, long)
10
11 def dict_iteritems(d):
12 return d.iteritems()
13
14 else:
15 int_types = int
16 unicode = str
17 xrange = range
18
19 def dict_iteritems(d):
20 return d.items()
21
22
23 if sys.version_info < (3, 5):
24 # Ugly hack...
25 RecursionError = RuntimeError
26
27 def _is_recursionerror(e):
28 return (
29 len(e.args) == 1
30 and isinstance(e.args[0], str)
31 and e.args[0].startswith("maximum recursion depth exceeded")
32 )
33
34 else:
35
36 def _is_recursionerror(e):
37 return True
38
39
40 if hasattr(sys, "pypy_version_info"):
41 # StringIO is slow on PyPy, StringIO is faster. However: PyPy's own
42 # StringBuilder is fastest.
43 from __pypy__ import newlist_hint
44
45 try:
46 from __pypy__.builders import BytesBuilder as StringBuilder
47 except ImportError:
48 from __pypy__.builders import StringBuilder
49 USING_STRINGBUILDER = True
50
51 class ESC[4;38;5;81mStringIO(ESC[4;38;5;149mobject):
52 def __init__(self, s=b""):
53 if s:
54 self.builder = StringBuilder(len(s))
55 self.builder.append(s)
56 else:
57 self.builder = StringBuilder()
58
59 def write(self, s):
60 if isinstance(s, memoryview):
61 s = s.tobytes()
62 elif isinstance(s, bytearray):
63 s = bytes(s)
64 self.builder.append(s)
65
66 def getvalue(self):
67 return self.builder.build()
68
69 else:
70 USING_STRINGBUILDER = False
71 from io import BytesIO as StringIO
72
73 newlist_hint = lambda size: []
74
75
76 from .exceptions import BufferFull, OutOfData, ExtraData, FormatError, StackError
77
78 from .ext import ExtType, Timestamp
79
80
81 EX_SKIP = 0
82 EX_CONSTRUCT = 1
83 EX_READ_ARRAY_HEADER = 2
84 EX_READ_MAP_HEADER = 3
85
86 TYPE_IMMEDIATE = 0
87 TYPE_ARRAY = 1
88 TYPE_MAP = 2
89 TYPE_RAW = 3
90 TYPE_BIN = 4
91 TYPE_EXT = 5
92
93 DEFAULT_RECURSE_LIMIT = 511
94
95
96 def _check_type_strict(obj, t, type=type, tuple=tuple):
97 if type(t) is tuple:
98 return type(obj) in t
99 else:
100 return type(obj) is t
101
102
103 def _get_data_from_buffer(obj):
104 view = memoryview(obj)
105 if view.itemsize != 1:
106 raise ValueError("cannot unpack from multi-byte object")
107 return view
108
109
110 def unpackb(packed, **kwargs):
111 """
112 Unpack an object from `packed`.
113
114 Raises ``ExtraData`` when *packed* contains extra bytes.
115 Raises ``ValueError`` when *packed* is incomplete.
116 Raises ``FormatError`` when *packed* is not valid msgpack.
117 Raises ``StackError`` when *packed* contains too nested.
118 Other exceptions can be raised during unpacking.
119
120 See :class:`Unpacker` for options.
121 """
122 unpacker = Unpacker(None, max_buffer_size=len(packed), **kwargs)
123 unpacker.feed(packed)
124 try:
125 ret = unpacker._unpack()
126 except OutOfData:
127 raise ValueError("Unpack failed: incomplete input")
128 except RecursionError as e:
129 if _is_recursionerror(e):
130 raise StackError
131 raise
132 if unpacker._got_extradata():
133 raise ExtraData(ret, unpacker._get_extradata())
134 return ret
135
136
137 if sys.version_info < (2, 7, 6):
138
139 def _unpack_from(f, b, o=0):
140 """Explicit type cast for legacy struct.unpack_from"""
141 return struct.unpack_from(f, bytes(b), o)
142
143 else:
144 _unpack_from = struct.unpack_from
145
146 _NO_FORMAT_USED = ""
147 _MSGPACK_HEADERS = {
148 0xC4: (1, _NO_FORMAT_USED, TYPE_BIN),
149 0xC5: (2, ">H", TYPE_BIN),
150 0xC6: (4, ">I", TYPE_BIN),
151 0xC7: (2, "Bb", TYPE_EXT),
152 0xC8: (3, ">Hb", TYPE_EXT),
153 0xC9: (5, ">Ib", TYPE_EXT),
154 0xCA: (4, ">f"),
155 0xCB: (8, ">d"),
156 0xCC: (1, _NO_FORMAT_USED),
157 0xCD: (2, ">H"),
158 0xCE: (4, ">I"),
159 0xCF: (8, ">Q"),
160 0xD0: (1, "b"),
161 0xD1: (2, ">h"),
162 0xD2: (4, ">i"),
163 0xD3: (8, ">q"),
164 0xD4: (1, "b1s", TYPE_EXT),
165 0xD5: (2, "b2s", TYPE_EXT),
166 0xD6: (4, "b4s", TYPE_EXT),
167 0xD7: (8, "b8s", TYPE_EXT),
168 0xD8: (16, "b16s", TYPE_EXT),
169 0xD9: (1, _NO_FORMAT_USED, TYPE_RAW),
170 0xDA: (2, ">H", TYPE_RAW),
171 0xDB: (4, ">I", TYPE_RAW),
172 0xDC: (2, ">H", TYPE_ARRAY),
173 0xDD: (4, ">I", TYPE_ARRAY),
174 0xDE: (2, ">H", TYPE_MAP),
175 0xDF: (4, ">I", TYPE_MAP),
176 }
177
178
179 class ESC[4;38;5;81mUnpacker(ESC[4;38;5;149mobject):
180 """Streaming unpacker.
181
182 Arguments:
183
184 :param file_like:
185 File-like object having `.read(n)` method.
186 If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
187
188 :param int read_size:
189 Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
190
191 :param bool use_list:
192 If true, unpack msgpack array to Python list.
193 Otherwise, unpack to Python tuple. (default: True)
194
195 :param bool raw:
196 If true, unpack msgpack raw to Python bytes.
197 Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
198
199 :param int timestamp:
200 Control how timestamp type is unpacked:
201
202 0 - Timestamp
203 1 - float (Seconds from the EPOCH)
204 2 - int (Nanoseconds from the EPOCH)
205 3 - datetime.datetime (UTC). Python 2 is not supported.
206
207 :param bool strict_map_key:
208 If true (default), only str or bytes are accepted for map (dict) keys.
209
210 :param callable object_hook:
211 When specified, it should be callable.
212 Unpacker calls it with a dict argument after unpacking msgpack map.
213 (See also simplejson)
214
215 :param callable object_pairs_hook:
216 When specified, it should be callable.
217 Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
218 (See also simplejson)
219
220 :param str unicode_errors:
221 The error handler for decoding unicode. (default: 'strict')
222 This option should be used only when you have msgpack data which
223 contains invalid UTF-8 string.
224
225 :param int max_buffer_size:
226 Limits size of data waiting unpacked. 0 means 2**32-1.
227 The default value is 100*1024*1024 (100MiB).
228 Raises `BufferFull` exception when it is insufficient.
229 You should set this parameter when unpacking data from untrusted source.
230
231 :param int max_str_len:
232 Deprecated, use *max_buffer_size* instead.
233 Limits max length of str. (default: max_buffer_size)
234
235 :param int max_bin_len:
236 Deprecated, use *max_buffer_size* instead.
237 Limits max length of bin. (default: max_buffer_size)
238
239 :param int max_array_len:
240 Limits max length of array.
241 (default: max_buffer_size)
242
243 :param int max_map_len:
244 Limits max length of map.
245 (default: max_buffer_size//2)
246
247 :param int max_ext_len:
248 Deprecated, use *max_buffer_size* instead.
249 Limits max size of ext type. (default: max_buffer_size)
250
251 Example of streaming deserialize from file-like object::
252
253 unpacker = Unpacker(file_like)
254 for o in unpacker:
255 process(o)
256
257 Example of streaming deserialize from socket::
258
259 unpacker = Unpacker()
260 while True:
261 buf = sock.recv(1024**2)
262 if not buf:
263 break
264 unpacker.feed(buf)
265 for o in unpacker:
266 process(o)
267
268 Raises ``ExtraData`` when *packed* contains extra bytes.
269 Raises ``OutOfData`` when *packed* is incomplete.
270 Raises ``FormatError`` when *packed* is not valid msgpack.
271 Raises ``StackError`` when *packed* contains too nested.
272 Other exceptions can be raised during unpacking.
273 """
274
275 def __init__(
276 self,
277 file_like=None,
278 read_size=0,
279 use_list=True,
280 raw=False,
281 timestamp=0,
282 strict_map_key=True,
283 object_hook=None,
284 object_pairs_hook=None,
285 list_hook=None,
286 unicode_errors=None,
287 max_buffer_size=100 * 1024 * 1024,
288 ext_hook=ExtType,
289 max_str_len=-1,
290 max_bin_len=-1,
291 max_array_len=-1,
292 max_map_len=-1,
293 max_ext_len=-1,
294 ):
295 if unicode_errors is None:
296 unicode_errors = "strict"
297
298 if file_like is None:
299 self._feeding = True
300 else:
301 if not callable(file_like.read):
302 raise TypeError("`file_like.read` must be callable")
303 self.file_like = file_like
304 self._feeding = False
305
306 #: array of bytes fed.
307 self._buffer = bytearray()
308 #: Which position we currently reads
309 self._buff_i = 0
310
311 # When Unpacker is used as an iterable, between the calls to next(),
312 # the buffer is not "consumed" completely, for efficiency sake.
313 # Instead, it is done sloppily. To make sure we raise BufferFull at
314 # the correct moments, we have to keep track of how sloppy we were.
315 # Furthermore, when the buffer is incomplete (that is: in the case
316 # we raise an OutOfData) we need to rollback the buffer to the correct
317 # state, which _buf_checkpoint records.
318 self._buf_checkpoint = 0
319
320 if not max_buffer_size:
321 max_buffer_size = 2**31 - 1
322 if max_str_len == -1:
323 max_str_len = max_buffer_size
324 if max_bin_len == -1:
325 max_bin_len = max_buffer_size
326 if max_array_len == -1:
327 max_array_len = max_buffer_size
328 if max_map_len == -1:
329 max_map_len = max_buffer_size // 2
330 if max_ext_len == -1:
331 max_ext_len = max_buffer_size
332
333 self._max_buffer_size = max_buffer_size
334 if read_size > self._max_buffer_size:
335 raise ValueError("read_size must be smaller than max_buffer_size")
336 self._read_size = read_size or min(self._max_buffer_size, 16 * 1024)
337 self._raw = bool(raw)
338 self._strict_map_key = bool(strict_map_key)
339 self._unicode_errors = unicode_errors
340 self._use_list = use_list
341 if not (0 <= timestamp <= 3):
342 raise ValueError("timestamp must be 0..3")
343 self._timestamp = timestamp
344 self._list_hook = list_hook
345 self._object_hook = object_hook
346 self._object_pairs_hook = object_pairs_hook
347 self._ext_hook = ext_hook
348 self._max_str_len = max_str_len
349 self._max_bin_len = max_bin_len
350 self._max_array_len = max_array_len
351 self._max_map_len = max_map_len
352 self._max_ext_len = max_ext_len
353 self._stream_offset = 0
354
355 if list_hook is not None and not callable(list_hook):
356 raise TypeError("`list_hook` is not callable")
357 if object_hook is not None and not callable(object_hook):
358 raise TypeError("`object_hook` is not callable")
359 if object_pairs_hook is not None and not callable(object_pairs_hook):
360 raise TypeError("`object_pairs_hook` is not callable")
361 if object_hook is not None and object_pairs_hook is not None:
362 raise TypeError(
363 "object_pairs_hook and object_hook are mutually " "exclusive"
364 )
365 if not callable(ext_hook):
366 raise TypeError("`ext_hook` is not callable")
367
368 def feed(self, next_bytes):
369 assert self._feeding
370 view = _get_data_from_buffer(next_bytes)
371 if len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size:
372 raise BufferFull
373
374 # Strip buffer before checkpoint before reading file.
375 if self._buf_checkpoint > 0:
376 del self._buffer[: self._buf_checkpoint]
377 self._buff_i -= self._buf_checkpoint
378 self._buf_checkpoint = 0
379
380 # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython
381 self._buffer.extend(view)
382
383 def _consume(self):
384 """Gets rid of the used parts of the buffer."""
385 self._stream_offset += self._buff_i - self._buf_checkpoint
386 self._buf_checkpoint = self._buff_i
387
388 def _got_extradata(self):
389 return self._buff_i < len(self._buffer)
390
391 def _get_extradata(self):
392 return self._buffer[self._buff_i :]
393
394 def read_bytes(self, n):
395 ret = self._read(n, raise_outofdata=False)
396 self._consume()
397 return ret
398
399 def _read(self, n, raise_outofdata=True):
400 # (int) -> bytearray
401 self._reserve(n, raise_outofdata=raise_outofdata)
402 i = self._buff_i
403 ret = self._buffer[i : i + n]
404 self._buff_i = i + len(ret)
405 return ret
406
407 def _reserve(self, n, raise_outofdata=True):
408 remain_bytes = len(self._buffer) - self._buff_i - n
409
410 # Fast path: buffer has n bytes already
411 if remain_bytes >= 0:
412 return
413
414 if self._feeding:
415 self._buff_i = self._buf_checkpoint
416 raise OutOfData
417
418 # Strip buffer before checkpoint before reading file.
419 if self._buf_checkpoint > 0:
420 del self._buffer[: self._buf_checkpoint]
421 self._buff_i -= self._buf_checkpoint
422 self._buf_checkpoint = 0
423
424 # Read from file
425 remain_bytes = -remain_bytes
426 if remain_bytes + len(self._buffer) > self._max_buffer_size:
427 raise BufferFull
428 while remain_bytes > 0:
429 to_read_bytes = max(self._read_size, remain_bytes)
430 read_data = self.file_like.read(to_read_bytes)
431 if not read_data:
432 break
433 assert isinstance(read_data, bytes)
434 self._buffer += read_data
435 remain_bytes -= len(read_data)
436
437 if len(self._buffer) < n + self._buff_i and raise_outofdata:
438 self._buff_i = 0 # rollback
439 raise OutOfData
440
441 def _read_header(self):
442 typ = TYPE_IMMEDIATE
443 n = 0
444 obj = None
445 self._reserve(1)
446 b = self._buffer[self._buff_i]
447 self._buff_i += 1
448 if b & 0b10000000 == 0:
449 obj = b
450 elif b & 0b11100000 == 0b11100000:
451 obj = -1 - (b ^ 0xFF)
452 elif b & 0b11100000 == 0b10100000:
453 n = b & 0b00011111
454 typ = TYPE_RAW
455 if n > self._max_str_len:
456 raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
457 obj = self._read(n)
458 elif b & 0b11110000 == 0b10010000:
459 n = b & 0b00001111
460 typ = TYPE_ARRAY
461 if n > self._max_array_len:
462 raise ValueError(
463 "%s exceeds max_array_len(%s)" % (n, self._max_array_len)
464 )
465 elif b & 0b11110000 == 0b10000000:
466 n = b & 0b00001111
467 typ = TYPE_MAP
468 if n > self._max_map_len:
469 raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
470 elif b == 0xC0:
471 obj = None
472 elif b == 0xC2:
473 obj = False
474 elif b == 0xC3:
475 obj = True
476 elif 0xC4 <= b <= 0xC6:
477 size, fmt, typ = _MSGPACK_HEADERS[b]
478 self._reserve(size)
479 if len(fmt) > 0:
480 n = _unpack_from(fmt, self._buffer, self._buff_i)[0]
481 else:
482 n = self._buffer[self._buff_i]
483 self._buff_i += size
484 if n > self._max_bin_len:
485 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
486 obj = self._read(n)
487 elif 0xC7 <= b <= 0xC9:
488 size, fmt, typ = _MSGPACK_HEADERS[b]
489 self._reserve(size)
490 L, n = _unpack_from(fmt, self._buffer, self._buff_i)
491 self._buff_i += size
492 if L > self._max_ext_len:
493 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
494 obj = self._read(L)
495 elif 0xCA <= b <= 0xD3:
496 size, fmt = _MSGPACK_HEADERS[b]
497 self._reserve(size)
498 if len(fmt) > 0:
499 obj = _unpack_from(fmt, self._buffer, self._buff_i)[0]
500 else:
501 obj = self._buffer[self._buff_i]
502 self._buff_i += size
503 elif 0xD4 <= b <= 0xD8:
504 size, fmt, typ = _MSGPACK_HEADERS[b]
505 if self._max_ext_len < size:
506 raise ValueError(
507 "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len)
508 )
509 self._reserve(size + 1)
510 n, obj = _unpack_from(fmt, self._buffer, self._buff_i)
511 self._buff_i += size + 1
512 elif 0xD9 <= b <= 0xDB:
513 size, fmt, typ = _MSGPACK_HEADERS[b]
514 self._reserve(size)
515 if len(fmt) > 0:
516 (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
517 else:
518 n = self._buffer[self._buff_i]
519 self._buff_i += size
520 if n > self._max_str_len:
521 raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
522 obj = self._read(n)
523 elif 0xDC <= b <= 0xDD:
524 size, fmt, typ = _MSGPACK_HEADERS[b]
525 self._reserve(size)
526 (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
527 self._buff_i += size
528 if n > self._max_array_len:
529 raise ValueError(
530 "%s exceeds max_array_len(%s)" % (n, self._max_array_len)
531 )
532 elif 0xDE <= b <= 0xDF:
533 size, fmt, typ = _MSGPACK_HEADERS[b]
534 self._reserve(size)
535 (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
536 self._buff_i += size
537 if n > self._max_map_len:
538 raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
539 else:
540 raise FormatError("Unknown header: 0x%x" % b)
541 return typ, n, obj
542
543 def _unpack(self, execute=EX_CONSTRUCT):
544 typ, n, obj = self._read_header()
545
546 if execute == EX_READ_ARRAY_HEADER:
547 if typ != TYPE_ARRAY:
548 raise ValueError("Expected array")
549 return n
550 if execute == EX_READ_MAP_HEADER:
551 if typ != TYPE_MAP:
552 raise ValueError("Expected map")
553 return n
554 # TODO should we eliminate the recursion?
555 if typ == TYPE_ARRAY:
556 if execute == EX_SKIP:
557 for i in xrange(n):
558 # TODO check whether we need to call `list_hook`
559 self._unpack(EX_SKIP)
560 return
561 ret = newlist_hint(n)
562 for i in xrange(n):
563 ret.append(self._unpack(EX_CONSTRUCT))
564 if self._list_hook is not None:
565 ret = self._list_hook(ret)
566 # TODO is the interaction between `list_hook` and `use_list` ok?
567 return ret if self._use_list else tuple(ret)
568 if typ == TYPE_MAP:
569 if execute == EX_SKIP:
570 for i in xrange(n):
571 # TODO check whether we need to call hooks
572 self._unpack(EX_SKIP)
573 self._unpack(EX_SKIP)
574 return
575 if self._object_pairs_hook is not None:
576 ret = self._object_pairs_hook(
577 (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT))
578 for _ in xrange(n)
579 )
580 else:
581 ret = {}
582 for _ in xrange(n):
583 key = self._unpack(EX_CONSTRUCT)
584 if self._strict_map_key and type(key) not in (unicode, bytes):
585 raise ValueError(
586 "%s is not allowed for map key" % str(type(key))
587 )
588 if not PY2 and type(key) is str:
589 key = sys.intern(key)
590 ret[key] = self._unpack(EX_CONSTRUCT)
591 if self._object_hook is not None:
592 ret = self._object_hook(ret)
593 return ret
594 if execute == EX_SKIP:
595 return
596 if typ == TYPE_RAW:
597 if self._raw:
598 obj = bytes(obj)
599 else:
600 obj = obj.decode("utf_8", self._unicode_errors)
601 return obj
602 if typ == TYPE_BIN:
603 return bytes(obj)
604 if typ == TYPE_EXT:
605 if n == -1: # timestamp
606 ts = Timestamp.from_bytes(bytes(obj))
607 if self._timestamp == 1:
608 return ts.to_unix()
609 elif self._timestamp == 2:
610 return ts.to_unix_nano()
611 elif self._timestamp == 3:
612 return ts.to_datetime()
613 else:
614 return ts
615 else:
616 return self._ext_hook(n, bytes(obj))
617 assert typ == TYPE_IMMEDIATE
618 return obj
619
620 def __iter__(self):
621 return self
622
623 def __next__(self):
624 try:
625 ret = self._unpack(EX_CONSTRUCT)
626 self._consume()
627 return ret
628 except OutOfData:
629 self._consume()
630 raise StopIteration
631 except RecursionError:
632 raise StackError
633
634 next = __next__
635
636 def skip(self):
637 self._unpack(EX_SKIP)
638 self._consume()
639
640 def unpack(self):
641 try:
642 ret = self._unpack(EX_CONSTRUCT)
643 except RecursionError:
644 raise StackError
645 self._consume()
646 return ret
647
648 def read_array_header(self):
649 ret = self._unpack(EX_READ_ARRAY_HEADER)
650 self._consume()
651 return ret
652
653 def read_map_header(self):
654 ret = self._unpack(EX_READ_MAP_HEADER)
655 self._consume()
656 return ret
657
658 def tell(self):
659 return self._stream_offset
660
661
662 class ESC[4;38;5;81mPacker(ESC[4;38;5;149mobject):
663 """
664 MessagePack Packer
665
666 Usage::
667
668 packer = Packer()
669 astream.write(packer.pack(a))
670 astream.write(packer.pack(b))
671
672 Packer's constructor has some keyword arguments:
673
674 :param callable default:
675 Convert user type to builtin type that Packer supports.
676 See also simplejson's document.
677
678 :param bool use_single_float:
679 Use single precision float type for float. (default: False)
680
681 :param bool autoreset:
682 Reset buffer after each pack and return its content as `bytes`. (default: True).
683 If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
684
685 :param bool use_bin_type:
686 Use bin type introduced in msgpack spec 2.0 for bytes.
687 It also enables str8 type for unicode. (default: True)
688
689 :param bool strict_types:
690 If set to true, types will be checked to be exact. Derived classes
691 from serializable types will not be serialized and will be
692 treated as unsupported type and forwarded to default.
693 Additionally tuples will not be serialized as lists.
694 This is useful when trying to implement accurate serialization
695 for python types.
696
697 :param bool datetime:
698 If set to true, datetime with tzinfo is packed into Timestamp type.
699 Note that the tzinfo is stripped in the timestamp.
700 You can get UTC datetime with `timestamp=3` option of the Unpacker.
701 (Python 2 is not supported).
702
703 :param str unicode_errors:
704 The error handler for encoding unicode. (default: 'strict')
705 DO NOT USE THIS!! This option is kept for very specific usage.
706
707 Example of streaming deserialize from file-like object::
708
709 unpacker = Unpacker(file_like)
710 for o in unpacker:
711 process(o)
712
713 Example of streaming deserialize from socket::
714
715 unpacker = Unpacker()
716 while True:
717 buf = sock.recv(1024**2)
718 if not buf:
719 break
720 unpacker.feed(buf)
721 for o in unpacker:
722 process(o)
723
724 Raises ``ExtraData`` when *packed* contains extra bytes.
725 Raises ``OutOfData`` when *packed* is incomplete.
726 Raises ``FormatError`` when *packed* is not valid msgpack.
727 Raises ``StackError`` when *packed* contains too nested.
728 Other exceptions can be raised during unpacking.
729 """
730
731 def __init__(
732 self,
733 default=None,
734 use_single_float=False,
735 autoreset=True,
736 use_bin_type=True,
737 strict_types=False,
738 datetime=False,
739 unicode_errors=None,
740 ):
741 self._strict_types = strict_types
742 self._use_float = use_single_float
743 self._autoreset = autoreset
744 self._use_bin_type = use_bin_type
745 self._buffer = StringIO()
746 if PY2 and datetime:
747 raise ValueError("datetime is not supported in Python 2")
748 self._datetime = bool(datetime)
749 self._unicode_errors = unicode_errors or "strict"
750 if default is not None:
751 if not callable(default):
752 raise TypeError("default must be callable")
753 self._default = default
754
755 def _pack(
756 self,
757 obj,
758 nest_limit=DEFAULT_RECURSE_LIMIT,
759 check=isinstance,
760 check_type_strict=_check_type_strict,
761 ):
762 default_used = False
763 if self._strict_types:
764 check = check_type_strict
765 list_types = list
766 else:
767 list_types = (list, tuple)
768 while True:
769 if nest_limit < 0:
770 raise ValueError("recursion limit exceeded")
771 if obj is None:
772 return self._buffer.write(b"\xc0")
773 if check(obj, bool):
774 if obj:
775 return self._buffer.write(b"\xc3")
776 return self._buffer.write(b"\xc2")
777 if check(obj, int_types):
778 if 0 <= obj < 0x80:
779 return self._buffer.write(struct.pack("B", obj))
780 if -0x20 <= obj < 0:
781 return self._buffer.write(struct.pack("b", obj))
782 if 0x80 <= obj <= 0xFF:
783 return self._buffer.write(struct.pack("BB", 0xCC, obj))
784 if -0x80 <= obj < 0:
785 return self._buffer.write(struct.pack(">Bb", 0xD0, obj))
786 if 0xFF < obj <= 0xFFFF:
787 return self._buffer.write(struct.pack(">BH", 0xCD, obj))
788 if -0x8000 <= obj < -0x80:
789 return self._buffer.write(struct.pack(">Bh", 0xD1, obj))
790 if 0xFFFF < obj <= 0xFFFFFFFF:
791 return self._buffer.write(struct.pack(">BI", 0xCE, obj))
792 if -0x80000000 <= obj < -0x8000:
793 return self._buffer.write(struct.pack(">Bi", 0xD2, obj))
794 if 0xFFFFFFFF < obj <= 0xFFFFFFFFFFFFFFFF:
795 return self._buffer.write(struct.pack(">BQ", 0xCF, obj))
796 if -0x8000000000000000 <= obj < -0x80000000:
797 return self._buffer.write(struct.pack(">Bq", 0xD3, obj))
798 if not default_used and self._default is not None:
799 obj = self._default(obj)
800 default_used = True
801 continue
802 raise OverflowError("Integer value out of range")
803 if check(obj, (bytes, bytearray)):
804 n = len(obj)
805 if n >= 2**32:
806 raise ValueError("%s is too large" % type(obj).__name__)
807 self._pack_bin_header(n)
808 return self._buffer.write(obj)
809 if check(obj, unicode):
810 obj = obj.encode("utf-8", self._unicode_errors)
811 n = len(obj)
812 if n >= 2**32:
813 raise ValueError("String is too large")
814 self._pack_raw_header(n)
815 return self._buffer.write(obj)
816 if check(obj, memoryview):
817 n = obj.nbytes
818 if n >= 2**32:
819 raise ValueError("Memoryview is too large")
820 self._pack_bin_header(n)
821 return self._buffer.write(obj)
822 if check(obj, float):
823 if self._use_float:
824 return self._buffer.write(struct.pack(">Bf", 0xCA, obj))
825 return self._buffer.write(struct.pack(">Bd", 0xCB, obj))
826 if check(obj, (ExtType, Timestamp)):
827 if check(obj, Timestamp):
828 code = -1
829 data = obj.to_bytes()
830 else:
831 code = obj.code
832 data = obj.data
833 assert isinstance(code, int)
834 assert isinstance(data, bytes)
835 L = len(data)
836 if L == 1:
837 self._buffer.write(b"\xd4")
838 elif L == 2:
839 self._buffer.write(b"\xd5")
840 elif L == 4:
841 self._buffer.write(b"\xd6")
842 elif L == 8:
843 self._buffer.write(b"\xd7")
844 elif L == 16:
845 self._buffer.write(b"\xd8")
846 elif L <= 0xFF:
847 self._buffer.write(struct.pack(">BB", 0xC7, L))
848 elif L <= 0xFFFF:
849 self._buffer.write(struct.pack(">BH", 0xC8, L))
850 else:
851 self._buffer.write(struct.pack(">BI", 0xC9, L))
852 self._buffer.write(struct.pack("b", code))
853 self._buffer.write(data)
854 return
855 if check(obj, list_types):
856 n = len(obj)
857 self._pack_array_header(n)
858 for i in xrange(n):
859 self._pack(obj[i], nest_limit - 1)
860 return
861 if check(obj, dict):
862 return self._pack_map_pairs(
863 len(obj), dict_iteritems(obj), nest_limit - 1
864 )
865
866 if self._datetime and check(obj, _DateTime) and obj.tzinfo is not None:
867 obj = Timestamp.from_datetime(obj)
868 default_used = 1
869 continue
870
871 if not default_used and self._default is not None:
872 obj = self._default(obj)
873 default_used = 1
874 continue
875
876 if self._datetime and check(obj, _DateTime):
877 raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,))
878
879 raise TypeError("Cannot serialize %r" % (obj,))
880
881 def pack(self, obj):
882 try:
883 self._pack(obj)
884 except:
885 self._buffer = StringIO() # force reset
886 raise
887 if self._autoreset:
888 ret = self._buffer.getvalue()
889 self._buffer = StringIO()
890 return ret
891
892 def pack_map_pairs(self, pairs):
893 self._pack_map_pairs(len(pairs), pairs)
894 if self._autoreset:
895 ret = self._buffer.getvalue()
896 self._buffer = StringIO()
897 return ret
898
899 def pack_array_header(self, n):
900 if n >= 2**32:
901 raise ValueError
902 self._pack_array_header(n)
903 if self._autoreset:
904 ret = self._buffer.getvalue()
905 self._buffer = StringIO()
906 return ret
907
908 def pack_map_header(self, n):
909 if n >= 2**32:
910 raise ValueError
911 self._pack_map_header(n)
912 if self._autoreset:
913 ret = self._buffer.getvalue()
914 self._buffer = StringIO()
915 return ret
916
917 def pack_ext_type(self, typecode, data):
918 if not isinstance(typecode, int):
919 raise TypeError("typecode must have int type.")
920 if not 0 <= typecode <= 127:
921 raise ValueError("typecode should be 0-127")
922 if not isinstance(data, bytes):
923 raise TypeError("data must have bytes type")
924 L = len(data)
925 if L > 0xFFFFFFFF:
926 raise ValueError("Too large data")
927 if L == 1:
928 self._buffer.write(b"\xd4")
929 elif L == 2:
930 self._buffer.write(b"\xd5")
931 elif L == 4:
932 self._buffer.write(b"\xd6")
933 elif L == 8:
934 self._buffer.write(b"\xd7")
935 elif L == 16:
936 self._buffer.write(b"\xd8")
937 elif L <= 0xFF:
938 self._buffer.write(b"\xc7" + struct.pack("B", L))
939 elif L <= 0xFFFF:
940 self._buffer.write(b"\xc8" + struct.pack(">H", L))
941 else:
942 self._buffer.write(b"\xc9" + struct.pack(">I", L))
943 self._buffer.write(struct.pack("B", typecode))
944 self._buffer.write(data)
945
946 def _pack_array_header(self, n):
947 if n <= 0x0F:
948 return self._buffer.write(struct.pack("B", 0x90 + n))
949 if n <= 0xFFFF:
950 return self._buffer.write(struct.pack(">BH", 0xDC, n))
951 if n <= 0xFFFFFFFF:
952 return self._buffer.write(struct.pack(">BI", 0xDD, n))
953 raise ValueError("Array is too large")
954
955 def _pack_map_header(self, n):
956 if n <= 0x0F:
957 return self._buffer.write(struct.pack("B", 0x80 + n))
958 if n <= 0xFFFF:
959 return self._buffer.write(struct.pack(">BH", 0xDE, n))
960 if n <= 0xFFFFFFFF:
961 return self._buffer.write(struct.pack(">BI", 0xDF, n))
962 raise ValueError("Dict is too large")
963
964 def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
965 self._pack_map_header(n)
966 for (k, v) in pairs:
967 self._pack(k, nest_limit - 1)
968 self._pack(v, nest_limit - 1)
969
970 def _pack_raw_header(self, n):
971 if n <= 0x1F:
972 self._buffer.write(struct.pack("B", 0xA0 + n))
973 elif self._use_bin_type and n <= 0xFF:
974 self._buffer.write(struct.pack(">BB", 0xD9, n))
975 elif n <= 0xFFFF:
976 self._buffer.write(struct.pack(">BH", 0xDA, n))
977 elif n <= 0xFFFFFFFF:
978 self._buffer.write(struct.pack(">BI", 0xDB, n))
979 else:
980 raise ValueError("Raw is too large")
981
982 def _pack_bin_header(self, n):
983 if not self._use_bin_type:
984 return self._pack_raw_header(n)
985 elif n <= 0xFF:
986 return self._buffer.write(struct.pack(">BB", 0xC4, n))
987 elif n <= 0xFFFF:
988 return self._buffer.write(struct.pack(">BH", 0xC5, n))
989 elif n <= 0xFFFFFFFF:
990 return self._buffer.write(struct.pack(">BI", 0xC6, n))
991 else:
992 raise ValueError("Bin is too large")
993
994 def bytes(self):
995 """Return internal buffer contents as bytes object"""
996 return self._buffer.getvalue()
997
998 def reset(self):
999 """Reset internal buffer.
1000
1001 This method is useful only when autoreset=False.
1002 """
1003 self._buffer = StringIO()
1004
1005 def getbuffer(self):
1006 """Return view of internal buffer."""
1007 if USING_STRINGBUILDER or PY2:
1008 return memoryview(self.bytes())
1009 else:
1010 return self._buffer.getbuffer()
Browse
Usage
Versions