1 """Create portable serialized representations of Python objects.
2
3 See module copyreg for a mechanism for registering custom picklers.
4 See module pickletools source for extensive comments.
5
6 Classes:
7
8 Pickler
9 Unpickler
10
11 Functions:
12
13 dump(object, file)
14 dumps(object) -> string
15 load(file) -> object
16 loads(bytes) -> object
17
18 Misc variables:
19
20 __version__
21 format_version
22 compatible_formats
23
24 """
25
26 from types import FunctionType
27 from copyreg import dispatch_table
28 from copyreg import _extension_registry, _inverted_registry, _extension_cache
29 from itertools import islice
30 from functools import partial
31 import sys
32 from sys import maxsize
33 from struct import pack, unpack
34 import re
35 import io
36 import codecs
37 import _compat_pickle
38
39 __all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
40 "Unpickler", "dump", "dumps", "load", "loads"]
41
42 try:
43 from _pickle import PickleBuffer
44 __all__.append("PickleBuffer")
45 _HAVE_PICKLE_BUFFER = True
46 except ImportError:
47 _HAVE_PICKLE_BUFFER = False
48
49
50 # Shortcut for use in isinstance testing
51 bytes_types = (bytes, bytearray)
52
53 # These are purely informational; no code uses these.
54 format_version = "4.0" # File format version we write
55 compatible_formats = ["1.0", # Original protocol 0
56 "1.1", # Protocol 0 with INST added
57 "1.2", # Original protocol 1
58 "1.3", # Protocol 1 with BINFLOAT added
59 "2.0", # Protocol 2
60 "3.0", # Protocol 3
61 "4.0", # Protocol 4
62 "5.0", # Protocol 5
63 ] # Old format versions we can read
64
65 # This is the highest protocol number we know how to read.
66 HIGHEST_PROTOCOL = 5
67
68 # The protocol we write by default. May be less than HIGHEST_PROTOCOL.
69 # Only bump this if the oldest still supported version of Python already
70 # includes it.
71 DEFAULT_PROTOCOL = 4
72
73 class ESC[4;38;5;81mPickleError(ESC[4;38;5;149mException):
74 """A common base class for the other pickling exceptions."""
75 pass
76
77 class ESC[4;38;5;81mPicklingError(ESC[4;38;5;149mPickleError):
78 """This exception is raised when an unpicklable object is passed to the
79 dump() method.
80
81 """
82 pass
83
84 class ESC[4;38;5;81mUnpicklingError(ESC[4;38;5;149mPickleError):
85 """This exception is raised when there is a problem unpickling an object,
86 such as a security violation.
87
88 Note that other exceptions may also be raised during unpickling, including
89 (but not necessarily limited to) AttributeError, EOFError, ImportError,
90 and IndexError.
91
92 """
93 pass
94
95 # An instance of _Stop is raised by Unpickler.load_stop() in response to
96 # the STOP opcode, passing the object that is the result of unpickling.
97 class ESC[4;38;5;81m_Stop(ESC[4;38;5;149mException):
98 def __init__(self, value):
99 self.value = value
100
101 # Jython has PyStringMap; it's a dict subclass with string keys
102 try:
103 from org.python.core import PyStringMap
104 except ImportError:
105 PyStringMap = None
106
107 # Pickle opcodes. See pickletools.py for extensive docs. The listing
108 # here is in kind-of alphabetical order of 1-character pickle code.
109 # pickletools groups them by purpose.
110
111 MARK = b'(' # push special markobject on stack
112 STOP = b'.' # every pickle ends with STOP
113 POP = b'0' # discard topmost stack item
114 POP_MARK = b'1' # discard stack top through topmost markobject
115 DUP = b'2' # duplicate top stack item
116 FLOAT = b'F' # push float object; decimal string argument
117 INT = b'I' # push integer or bool; decimal string argument
118 BININT = b'J' # push four-byte signed int
119 BININT1 = b'K' # push 1-byte unsigned int
120 LONG = b'L' # push long; decimal string argument
121 BININT2 = b'M' # push 2-byte unsigned int
122 NONE = b'N' # push None
123 PERSID = b'P' # push persistent object; id is taken from string arg
124 BINPERSID = b'Q' # " " " ; " " " " stack
125 REDUCE = b'R' # apply callable to argtuple, both on stack
126 STRING = b'S' # push string; NL-terminated string argument
127 BINSTRING = b'T' # push string; counted binary string argument
128 SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes
129 UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument
130 BINUNICODE = b'X' # " " " ; counted UTF-8 string argument
131 APPEND = b'a' # append stack top to list below it
132 BUILD = b'b' # call __setstate__ or __dict__.update()
133 GLOBAL = b'c' # push self.find_class(modname, name); 2 string args
134 DICT = b'd' # build a dict from stack items
135 EMPTY_DICT = b'}' # push empty dict
136 APPENDS = b'e' # extend list on stack by topmost stack slice
137 GET = b'g' # push item from memo on stack; index is string arg
138 BINGET = b'h' # " " " " " " ; " " 1-byte arg
139 INST = b'i' # build & push class instance
140 LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg
141 LIST = b'l' # build list from topmost stack items
142 EMPTY_LIST = b']' # push empty list
143 OBJ = b'o' # build & push class instance
144 PUT = b'p' # store stack top in memo; index is string arg
145 BINPUT = b'q' # " " " " " ; " " 1-byte arg
146 LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg
147 SETITEM = b's' # add key+value pair to dict
148 TUPLE = b't' # build tuple from topmost stack items
149 EMPTY_TUPLE = b')' # push empty tuple
150 SETITEMS = b'u' # modify dict by adding topmost key+value pairs
151 BINFLOAT = b'G' # push float; arg is 8-byte float encoding
152
153 TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py
154 FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py
155
156 # Protocol 2
157
158 PROTO = b'\x80' # identify pickle protocol
159 NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple
160 EXT1 = b'\x82' # push object from extension registry; 1-byte index
161 EXT2 = b'\x83' # ditto, but 2-byte index
162 EXT4 = b'\x84' # ditto, but 4-byte index
163 TUPLE1 = b'\x85' # build 1-tuple from stack top
164 TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items
165 TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items
166 NEWTRUE = b'\x88' # push True
167 NEWFALSE = b'\x89' # push False
168 LONG1 = b'\x8a' # push long from < 256 bytes
169 LONG4 = b'\x8b' # push really big long
170
171 _tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
172
173 # Protocol 3 (Python 3.x)
174
175 BINBYTES = b'B' # push bytes; counted binary string argument
176 SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
177
178 # Protocol 4
179
180 SHORT_BINUNICODE = b'\x8c' # push short string; UTF-8 length < 256 bytes
181 BINUNICODE8 = b'\x8d' # push very long string
182 BINBYTES8 = b'\x8e' # push very long bytes string
183 EMPTY_SET = b'\x8f' # push empty set on the stack
184 ADDITEMS = b'\x90' # modify set by adding topmost stack items
185 FROZENSET = b'\x91' # build frozenset from topmost stack items
186 NEWOBJ_EX = b'\x92' # like NEWOBJ but work with keyword only arguments
187 STACK_GLOBAL = b'\x93' # same as GLOBAL but using names on the stacks
188 MEMOIZE = b'\x94' # store top of the stack in memo
189 FRAME = b'\x95' # indicate the beginning of a new frame
190
191 # Protocol 5
192
193 BYTEARRAY8 = b'\x96' # push bytearray
194 NEXT_BUFFER = b'\x97' # push next out-of-band buffer
195 READONLY_BUFFER = b'\x98' # make top of stack readonly
196
197 __all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)])
198
199
200 class ESC[4;38;5;81m_Framer:
201
202 _FRAME_SIZE_MIN = 4
203 _FRAME_SIZE_TARGET = 64 * 1024
204
205 def __init__(self, file_write):
206 self.file_write = file_write
207 self.current_frame = None
208
209 def start_framing(self):
210 self.current_frame = io.BytesIO()
211
212 def end_framing(self):
213 if self.current_frame and self.current_frame.tell() > 0:
214 self.commit_frame(force=True)
215 self.current_frame = None
216
217 def commit_frame(self, force=False):
218 if self.current_frame:
219 f = self.current_frame
220 if f.tell() >= self._FRAME_SIZE_TARGET or force:
221 data = f.getbuffer()
222 write = self.file_write
223 if len(data) >= self._FRAME_SIZE_MIN:
224 # Issue a single call to the write method of the underlying
225 # file object for the frame opcode with the size of the
226 # frame. The concatenation is expected to be less expensive
227 # than issuing an additional call to write.
228 write(FRAME + pack("<Q", len(data)))
229
230 # Issue a separate call to write to append the frame
231 # contents without concatenation to the above to avoid a
232 # memory copy.
233 write(data)
234
235 # Start the new frame with a new io.BytesIO instance so that
236 # the file object can have delayed access to the previous frame
237 # contents via an unreleased memoryview of the previous
238 # io.BytesIO instance.
239 self.current_frame = io.BytesIO()
240
241 def write(self, data):
242 if self.current_frame:
243 return self.current_frame.write(data)
244 else:
245 return self.file_write(data)
246
247 def write_large_bytes(self, header, payload):
248 write = self.file_write
249 if self.current_frame:
250 # Terminate the current frame and flush it to the file.
251 self.commit_frame(force=True)
252
253 # Perform direct write of the header and payload of the large binary
254 # object. Be careful not to concatenate the header and the payload
255 # prior to calling 'write' as we do not want to allocate a large
256 # temporary bytes object.
257 # We intentionally do not insert a protocol 4 frame opcode to make
258 # it possible to optimize file.read calls in the loader.
259 write(header)
260 write(payload)
261
262
263 class ESC[4;38;5;81m_Unframer:
264
265 def __init__(self, file_read, file_readline, file_tell=None):
266 self.file_read = file_read
267 self.file_readline = file_readline
268 self.current_frame = None
269
270 def readinto(self, buf):
271 if self.current_frame:
272 n = self.current_frame.readinto(buf)
273 if n == 0 and len(buf) != 0:
274 self.current_frame = None
275 n = len(buf)
276 buf[:] = self.file_read(n)
277 return n
278 if n < len(buf):
279 raise UnpicklingError(
280 "pickle exhausted before end of frame")
281 return n
282 else:
283 n = len(buf)
284 buf[:] = self.file_read(n)
285 return n
286
287 def read(self, n):
288 if self.current_frame:
289 data = self.current_frame.read(n)
290 if not data and n != 0:
291 self.current_frame = None
292 return self.file_read(n)
293 if len(data) < n:
294 raise UnpicklingError(
295 "pickle exhausted before end of frame")
296 return data
297 else:
298 return self.file_read(n)
299
300 def readline(self):
301 if self.current_frame:
302 data = self.current_frame.readline()
303 if not data:
304 self.current_frame = None
305 return self.file_readline()
306 if data[-1] != b'\n'[0]:
307 raise UnpicklingError(
308 "pickle exhausted before end of frame")
309 return data
310 else:
311 return self.file_readline()
312
313 def load_frame(self, frame_size):
314 if self.current_frame and self.current_frame.read() != b'':
315 raise UnpicklingError(
316 "beginning of a new frame before end of current frame")
317 self.current_frame = io.BytesIO(self.file_read(frame_size))
318
319
320 # Tools used for pickling.
321
322 def _getattribute(obj, name):
323 for subpath in name.split('.'):
324 if subpath == '<locals>':
325 raise AttributeError("Can't get local attribute {!r} on {!r}"
326 .format(name, obj))
327 try:
328 parent = obj
329 obj = getattr(obj, subpath)
330 except AttributeError:
331 raise AttributeError("Can't get attribute {!r} on {!r}"
332 .format(name, obj)) from None
333 return obj, parent
334
335 def whichmodule(obj, name):
336 """Find the module an object belong to."""
337 module_name = getattr(obj, '__module__', None)
338 if module_name is not None:
339 return module_name
340 # Protect the iteration by using a list copy of sys.modules against dynamic
341 # modules that trigger imports of other modules upon calls to getattr.
342 for module_name, module in sys.modules.copy().items():
343 if (module_name == '__main__'
344 or module_name == '__mp_main__' # bpo-42406
345 or module is None):
346 continue
347 try:
348 if _getattribute(module, name)[0] is obj:
349 return module_name
350 except AttributeError:
351 pass
352 return '__main__'
353
354 def encode_long(x):
355 r"""Encode a long to a two's complement little-endian binary string.
356 Note that 0 is a special case, returning an empty string, to save a
357 byte in the LONG1 pickling context.
358
359 >>> encode_long(0)
360 b''
361 >>> encode_long(255)
362 b'\xff\x00'
363 >>> encode_long(32767)
364 b'\xff\x7f'
365 >>> encode_long(-256)
366 b'\x00\xff'
367 >>> encode_long(-32768)
368 b'\x00\x80'
369 >>> encode_long(-128)
370 b'\x80'
371 >>> encode_long(127)
372 b'\x7f'
373 >>>
374 """
375 if x == 0:
376 return b''
377 nbytes = (x.bit_length() >> 3) + 1
378 result = x.to_bytes(nbytes, byteorder='little', signed=True)
379 if x < 0 and nbytes > 1:
380 if result[-1] == 0xff and (result[-2] & 0x80) != 0:
381 result = result[:-1]
382 return result
383
384 def decode_long(data):
385 r"""Decode a long from a two's complement little-endian binary string.
386
387 >>> decode_long(b'')
388 0
389 >>> decode_long(b"\xff\x00")
390 255
391 >>> decode_long(b"\xff\x7f")
392 32767
393 >>> decode_long(b"\x00\xff")
394 -256
395 >>> decode_long(b"\x00\x80")
396 -32768
397 >>> decode_long(b"\x80")
398 -128
399 >>> decode_long(b"\x7f")
400 127
401 """
402 return int.from_bytes(data, byteorder='little', signed=True)
403
404
405 # Pickling machinery
406
407 class ESC[4;38;5;81m_Pickler:
408
409 def __init__(self, file, protocol=None, *, fix_imports=True,
410 buffer_callback=None):
411 """This takes a binary file for writing a pickle data stream.
412
413 The optional *protocol* argument tells the pickler to use the
414 given protocol; supported protocols are 0, 1, 2, 3, 4 and 5.
415 The default protocol is 4. It was introduced in Python 3.4, and
416 is incompatible with previous versions.
417
418 Specifying a negative protocol version selects the highest
419 protocol version supported. The higher the protocol used, the
420 more recent the version of Python needed to read the pickle
421 produced.
422
423 The *file* argument must have a write() method that accepts a
424 single bytes argument. It can thus be a file object opened for
425 binary writing, an io.BytesIO instance, or any other custom
426 object that meets this interface.
427
428 If *fix_imports* is True and *protocol* is less than 3, pickle
429 will try to map the new Python 3 names to the old module names
430 used in Python 2, so that the pickle data stream is readable
431 with Python 2.
432
433 If *buffer_callback* is None (the default), buffer views are
434 serialized into *file* as part of the pickle stream.
435
436 If *buffer_callback* is not None, then it can be called any number
437 of times with a buffer view. If the callback returns a false value
438 (such as None), the given buffer is out-of-band; otherwise the
439 buffer is serialized in-band, i.e. inside the pickle stream.
440
441 It is an error if *buffer_callback* is not None and *protocol*
442 is None or smaller than 5.
443 """
444 if protocol is None:
445 protocol = DEFAULT_PROTOCOL
446 if protocol < 0:
447 protocol = HIGHEST_PROTOCOL
448 elif not 0 <= protocol <= HIGHEST_PROTOCOL:
449 raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
450 if buffer_callback is not None and protocol < 5:
451 raise ValueError("buffer_callback needs protocol >= 5")
452 self._buffer_callback = buffer_callback
453 try:
454 self._file_write = file.write
455 except AttributeError:
456 raise TypeError("file must have a 'write' attribute")
457 self.framer = _Framer(self._file_write)
458 self.write = self.framer.write
459 self._write_large_bytes = self.framer.write_large_bytes
460 self.memo = {}
461 self.proto = int(protocol)
462 self.bin = protocol >= 1
463 self.fast = 0
464 self.fix_imports = fix_imports and protocol < 3
465
466 def clear_memo(self):
467 """Clears the pickler's "memo".
468
469 The memo is the data structure that remembers which objects the
470 pickler has already seen, so that shared or recursive objects
471 are pickled by reference and not by value. This method is
472 useful when re-using picklers.
473 """
474 self.memo.clear()
475
476 def dump(self, obj):
477 """Write a pickled representation of obj to the open file."""
478 # Check whether Pickler was initialized correctly. This is
479 # only needed to mimic the behavior of _pickle.Pickler.dump().
480 if not hasattr(self, "_file_write"):
481 raise PicklingError("Pickler.__init__() was not called by "
482 "%s.__init__()" % (self.__class__.__name__,))
483 if self.proto >= 2:
484 self.write(PROTO + pack("<B", self.proto))
485 if self.proto >= 4:
486 self.framer.start_framing()
487 self.save(obj)
488 self.write(STOP)
489 self.framer.end_framing()
490
491 def memoize(self, obj):
492 """Store an object in the memo."""
493
494 # The Pickler memo is a dictionary mapping object ids to 2-tuples
495 # that contain the Unpickler memo key and the object being memoized.
496 # The memo key is written to the pickle and will become
497 # the key in the Unpickler's memo. The object is stored in the
498 # Pickler memo so that transient objects are kept alive during
499 # pickling.
500
501 # The use of the Unpickler memo length as the memo key is just a
502 # convention. The only requirement is that the memo values be unique.
503 # But there appears no advantage to any other scheme, and this
504 # scheme allows the Unpickler memo to be implemented as a plain (but
505 # growable) array, indexed by memo key.
506 if self.fast:
507 return
508 assert id(obj) not in self.memo
509 idx = len(self.memo)
510 self.write(self.put(idx))
511 self.memo[id(obj)] = idx, obj
512
513 # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
514 def put(self, idx):
515 if self.proto >= 4:
516 return MEMOIZE
517 elif self.bin:
518 if idx < 256:
519 return BINPUT + pack("<B", idx)
520 else:
521 return LONG_BINPUT + pack("<I", idx)
522 else:
523 return PUT + repr(idx).encode("ascii") + b'\n'
524
525 # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
526 def get(self, i):
527 if self.bin:
528 if i < 256:
529 return BINGET + pack("<B", i)
530 else:
531 return LONG_BINGET + pack("<I", i)
532
533 return GET + repr(i).encode("ascii") + b'\n'
534
535 def save(self, obj, save_persistent_id=True):
536 self.framer.commit_frame()
537
538 # Check for persistent id (defined by a subclass)
539 pid = self.persistent_id(obj)
540 if pid is not None and save_persistent_id:
541 self.save_pers(pid)
542 return
543
544 # Check the memo
545 x = self.memo.get(id(obj))
546 if x is not None:
547 self.write(self.get(x[0]))
548 return
549
550 rv = NotImplemented
551 reduce = getattr(self, "reducer_override", None)
552 if reduce is not None:
553 rv = reduce(obj)
554
555 if rv is NotImplemented:
556 # Check the type dispatch table
557 t = type(obj)
558 f = self.dispatch.get(t)
559 if f is not None:
560 f(self, obj) # Call unbound method with explicit self
561 return
562
563 # Check private dispatch table if any, or else
564 # copyreg.dispatch_table
565 reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
566 if reduce is not None:
567 rv = reduce(obj)
568 else:
569 # Check for a class with a custom metaclass; treat as regular
570 # class
571 if issubclass(t, type):
572 self.save_global(obj)
573 return
574
575 # Check for a __reduce_ex__ method, fall back to __reduce__
576 reduce = getattr(obj, "__reduce_ex__", None)
577 if reduce is not None:
578 rv = reduce(self.proto)
579 else:
580 reduce = getattr(obj, "__reduce__", None)
581 if reduce is not None:
582 rv = reduce()
583 else:
584 raise PicklingError("Can't pickle %r object: %r" %
585 (t.__name__, obj))
586
587 # Check for string returned by reduce(), meaning "save as global"
588 if isinstance(rv, str):
589 self.save_global(obj, rv)
590 return
591
592 # Assert that reduce() returned a tuple
593 if not isinstance(rv, tuple):
594 raise PicklingError("%s must return string or tuple" % reduce)
595
596 # Assert that it returned an appropriately sized tuple
597 l = len(rv)
598 if not (2 <= l <= 6):
599 raise PicklingError("Tuple returned by %s must have "
600 "two to six elements" % reduce)
601
602 # Save the reduce() output and finally memoize the object
603 self.save_reduce(obj=obj, *rv)
604
605 def persistent_id(self, obj):
606 # This exists so a subclass can override it
607 return None
608
609 def save_pers(self, pid):
610 # Save a persistent id reference
611 if self.bin:
612 self.save(pid, save_persistent_id=False)
613 self.write(BINPERSID)
614 else:
615 try:
616 self.write(PERSID + str(pid).encode("ascii") + b'\n')
617 except UnicodeEncodeError:
618 raise PicklingError(
619 "persistent IDs in protocol 0 must be ASCII strings")
620
621 def save_reduce(self, func, args, state=None, listitems=None,
622 dictitems=None, state_setter=None, *, obj=None):
623 # This API is called by some subclasses
624
625 if not isinstance(args, tuple):
626 raise PicklingError("args from save_reduce() must be a tuple")
627 if not callable(func):
628 raise PicklingError("func from save_reduce() must be callable")
629
630 save = self.save
631 write = self.write
632
633 func_name = getattr(func, "__name__", "")
634 if self.proto >= 2 and func_name == "__newobj_ex__":
635 cls, args, kwargs = args
636 if not hasattr(cls, "__new__"):
637 raise PicklingError("args[0] from {} args has no __new__"
638 .format(func_name))
639 if obj is not None and cls is not obj.__class__:
640 raise PicklingError("args[0] from {} args has the wrong class"
641 .format(func_name))
642 if self.proto >= 4:
643 save(cls)
644 save(args)
645 save(kwargs)
646 write(NEWOBJ_EX)
647 else:
648 func = partial(cls.__new__, cls, *args, **kwargs)
649 save(func)
650 save(())
651 write(REDUCE)
652 elif self.proto >= 2 and func_name == "__newobj__":
653 # A __reduce__ implementation can direct protocol 2 or newer to
654 # use the more efficient NEWOBJ opcode, while still
655 # allowing protocol 0 and 1 to work normally. For this to
656 # work, the function returned by __reduce__ should be
657 # called __newobj__, and its first argument should be a
658 # class. The implementation for __newobj__
659 # should be as follows, although pickle has no way to
660 # verify this:
661 #
662 # def __newobj__(cls, *args):
663 # return cls.__new__(cls, *args)
664 #
665 # Protocols 0 and 1 will pickle a reference to __newobj__,
666 # while protocol 2 (and above) will pickle a reference to
667 # cls, the remaining args tuple, and the NEWOBJ code,
668 # which calls cls.__new__(cls, *args) at unpickling time
669 # (see load_newobj below). If __reduce__ returns a
670 # three-tuple, the state from the third tuple item will be
671 # pickled regardless of the protocol, calling __setstate__
672 # at unpickling time (see load_build below).
673 #
674 # Note that no standard __newobj__ implementation exists;
675 # you have to provide your own. This is to enforce
676 # compatibility with Python 2.2 (pickles written using
677 # protocol 0 or 1 in Python 2.3 should be unpicklable by
678 # Python 2.2).
679 cls = args[0]
680 if not hasattr(cls, "__new__"):
681 raise PicklingError(
682 "args[0] from __newobj__ args has no __new__")
683 if obj is not None and cls is not obj.__class__:
684 raise PicklingError(
685 "args[0] from __newobj__ args has the wrong class")
686 args = args[1:]
687 save(cls)
688 save(args)
689 write(NEWOBJ)
690 else:
691 save(func)
692 save(args)
693 write(REDUCE)
694
695 if obj is not None:
696 # If the object is already in the memo, this means it is
697 # recursive. In this case, throw away everything we put on the
698 # stack, and fetch the object back from the memo.
699 if id(obj) in self.memo:
700 write(POP + self.get(self.memo[id(obj)][0]))
701 else:
702 self.memoize(obj)
703
704 # More new special cases (that work with older protocols as
705 # well): when __reduce__ returns a tuple with 4 or 5 items,
706 # the 4th and 5th item should be iterators that provide list
707 # items and dict items (as (key, value) tuples), or None.
708
709 if listitems is not None:
710 self._batch_appends(listitems)
711
712 if dictitems is not None:
713 self._batch_setitems(dictitems)
714
715 if state is not None:
716 if state_setter is None:
717 save(state)
718 write(BUILD)
719 else:
720 # If a state_setter is specified, call it instead of load_build
721 # to update obj's with its previous state.
722 # First, push state_setter and its tuple of expected arguments
723 # (obj, state) onto the stack.
724 save(state_setter)
725 save(obj) # simple BINGET opcode as obj is already memoized.
726 save(state)
727 write(TUPLE2)
728 # Trigger a state_setter(obj, state) function call.
729 write(REDUCE)
730 # The purpose of state_setter is to carry-out an
731 # inplace modification of obj. We do not care about what the
732 # method might return, so its output is eventually removed from
733 # the stack.
734 write(POP)
735
736 # Methods below this point are dispatched through the dispatch table
737
738 dispatch = {}
739
740 def save_none(self, obj):
741 self.write(NONE)
742 dispatch[type(None)] = save_none
743
744 def save_bool(self, obj):
745 if self.proto >= 2:
746 self.write(NEWTRUE if obj else NEWFALSE)
747 else:
748 self.write(TRUE if obj else FALSE)
749 dispatch[bool] = save_bool
750
751 def save_long(self, obj):
752 if self.bin:
753 # If the int is small enough to fit in a signed 4-byte 2's-comp
754 # format, we can store it more efficiently than the general
755 # case.
756 # First one- and two-byte unsigned ints:
757 if obj >= 0:
758 if obj <= 0xff:
759 self.write(BININT1 + pack("<B", obj))
760 return
761 if obj <= 0xffff:
762 self.write(BININT2 + pack("<H", obj))
763 return
764 # Next check for 4-byte signed ints:
765 if -0x80000000 <= obj <= 0x7fffffff:
766 self.write(BININT + pack("<i", obj))
767 return
768 if self.proto >= 2:
769 encoded = encode_long(obj)
770 n = len(encoded)
771 if n < 256:
772 self.write(LONG1 + pack("<B", n) + encoded)
773 else:
774 self.write(LONG4 + pack("<i", n) + encoded)
775 return
776 if -0x80000000 <= obj <= 0x7fffffff:
777 self.write(INT + repr(obj).encode("ascii") + b'\n')
778 else:
779 self.write(LONG + repr(obj).encode("ascii") + b'L\n')
780 dispatch[int] = save_long
781
782 def save_float(self, obj):
783 if self.bin:
784 self.write(BINFLOAT + pack('>d', obj))
785 else:
786 self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
787 dispatch[float] = save_float
788
789 def save_bytes(self, obj):
790 if self.proto < 3:
791 if not obj: # bytes object is empty
792 self.save_reduce(bytes, (), obj=obj)
793 else:
794 self.save_reduce(codecs.encode,
795 (str(obj, 'latin1'), 'latin1'), obj=obj)
796 return
797 n = len(obj)
798 if n <= 0xff:
799 self.write(SHORT_BINBYTES + pack("<B", n) + obj)
800 elif n > 0xffffffff and self.proto >= 4:
801 self._write_large_bytes(BINBYTES8 + pack("<Q", n), obj)
802 elif n >= self.framer._FRAME_SIZE_TARGET:
803 self._write_large_bytes(BINBYTES + pack("<I", n), obj)
804 else:
805 self.write(BINBYTES + pack("<I", n) + obj)
806 self.memoize(obj)
807 dispatch[bytes] = save_bytes
808
809 def save_bytearray(self, obj):
810 if self.proto < 5:
811 if not obj: # bytearray is empty
812 self.save_reduce(bytearray, (), obj=obj)
813 else:
814 self.save_reduce(bytearray, (bytes(obj),), obj=obj)
815 return
816 n = len(obj)
817 if n >= self.framer._FRAME_SIZE_TARGET:
818 self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
819 else:
820 self.write(BYTEARRAY8 + pack("<Q", n) + obj)
821 self.memoize(obj)
822 dispatch[bytearray] = save_bytearray
823
824 if _HAVE_PICKLE_BUFFER:
825 def save_picklebuffer(self, obj):
826 if self.proto < 5:
827 raise PicklingError("PickleBuffer can only pickled with "
828 "protocol >= 5")
829 with obj.raw() as m:
830 if not m.contiguous:
831 raise PicklingError("PickleBuffer can not be pickled when "
832 "pointing to a non-contiguous buffer")
833 in_band = True
834 if self._buffer_callback is not None:
835 in_band = bool(self._buffer_callback(obj))
836 if in_band:
837 # Write data in-band
838 # XXX The C implementation avoids a copy here
839 if m.readonly:
840 self.save_bytes(m.tobytes())
841 else:
842 self.save_bytearray(m.tobytes())
843 else:
844 # Write data out-of-band
845 self.write(NEXT_BUFFER)
846 if m.readonly:
847 self.write(READONLY_BUFFER)
848
849 dispatch[PickleBuffer] = save_picklebuffer
850
851 def save_str(self, obj):
852 if self.bin:
853 encoded = obj.encode('utf-8', 'surrogatepass')
854 n = len(encoded)
855 if n <= 0xff and self.proto >= 4:
856 self.write(SHORT_BINUNICODE + pack("<B", n) + encoded)
857 elif n > 0xffffffff and self.proto >= 4:
858 self._write_large_bytes(BINUNICODE8 + pack("<Q", n), encoded)
859 elif n >= self.framer._FRAME_SIZE_TARGET:
860 self._write_large_bytes(BINUNICODE + pack("<I", n), encoded)
861 else:
862 self.write(BINUNICODE + pack("<I", n) + encoded)
863 else:
864 obj = obj.replace("\\", "\\u005c")
865 obj = obj.replace("\0", "\\u0000")
866 obj = obj.replace("\n", "\\u000a")
867 obj = obj.replace("\r", "\\u000d")
868 obj = obj.replace("\x1a", "\\u001a") # EOF on DOS
869 self.write(UNICODE + obj.encode('raw-unicode-escape') +
870 b'\n')
871 self.memoize(obj)
872 dispatch[str] = save_str
873
874 def save_tuple(self, obj):
875 if not obj: # tuple is empty
876 if self.bin:
877 self.write(EMPTY_TUPLE)
878 else:
879 self.write(MARK + TUPLE)
880 return
881
882 n = len(obj)
883 save = self.save
884 memo = self.memo
885 if n <= 3 and self.proto >= 2:
886 for element in obj:
887 save(element)
888 # Subtle. Same as in the big comment below.
889 if id(obj) in memo:
890 get = self.get(memo[id(obj)][0])
891 self.write(POP * n + get)
892 else:
893 self.write(_tuplesize2code[n])
894 self.memoize(obj)
895 return
896
897 # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
898 # has more than 3 elements.
899 write = self.write
900 write(MARK)
901 for element in obj:
902 save(element)
903
904 if id(obj) in memo:
905 # Subtle. d was not in memo when we entered save_tuple(), so
906 # the process of saving the tuple's elements must have saved
907 # the tuple itself: the tuple is recursive. The proper action
908 # now is to throw away everything we put on the stack, and
909 # simply GET the tuple (it's already constructed). This check
910 # could have been done in the "for element" loop instead, but
911 # recursive tuples are a rare thing.
912 get = self.get(memo[id(obj)][0])
913 if self.bin:
914 write(POP_MARK + get)
915 else: # proto 0 -- POP_MARK not available
916 write(POP * (n+1) + get)
917 return
918
919 # No recursion.
920 write(TUPLE)
921 self.memoize(obj)
922
923 dispatch[tuple] = save_tuple
924
925 def save_list(self, obj):
926 if self.bin:
927 self.write(EMPTY_LIST)
928 else: # proto 0 -- can't use EMPTY_LIST
929 self.write(MARK + LIST)
930
931 self.memoize(obj)
932 self._batch_appends(obj)
933
934 dispatch[list] = save_list
935
936 _BATCHSIZE = 1000
937
938 def _batch_appends(self, items):
939 # Helper to batch up APPENDS sequences
940 save = self.save
941 write = self.write
942
943 if not self.bin:
944 for x in items:
945 save(x)
946 write(APPEND)
947 return
948
949 it = iter(items)
950 while True:
951 tmp = list(islice(it, self._BATCHSIZE))
952 n = len(tmp)
953 if n > 1:
954 write(MARK)
955 for x in tmp:
956 save(x)
957 write(APPENDS)
958 elif n:
959 save(tmp[0])
960 write(APPEND)
961 # else tmp is empty, and we're done
962 if n < self._BATCHSIZE:
963 return
964
965 def save_dict(self, obj):
966 if self.bin:
967 self.write(EMPTY_DICT)
968 else: # proto 0 -- can't use EMPTY_DICT
969 self.write(MARK + DICT)
970
971 self.memoize(obj)
972 self._batch_setitems(obj.items())
973
974 dispatch[dict] = save_dict
975 if PyStringMap is not None:
976 dispatch[PyStringMap] = save_dict
977
978 def _batch_setitems(self, items):
979 # Helper to batch up SETITEMS sequences; proto >= 1 only
980 save = self.save
981 write = self.write
982
983 if not self.bin:
984 for k, v in items:
985 save(k)
986 save(v)
987 write(SETITEM)
988 return
989
990 it = iter(items)
991 while True:
992 tmp = list(islice(it, self._BATCHSIZE))
993 n = len(tmp)
994 if n > 1:
995 write(MARK)
996 for k, v in tmp:
997 save(k)
998 save(v)
999 write(SETITEMS)
1000 elif n:
1001 k, v = tmp[0]
1002 save(k)
1003 save(v)
1004 write(SETITEM)
1005 # else tmp is empty, and we're done
1006 if n < self._BATCHSIZE:
1007 return
1008
1009 def save_set(self, obj):
1010 save = self.save
1011 write = self.write
1012
1013 if self.proto < 4:
1014 self.save_reduce(set, (list(obj),), obj=obj)
1015 return
1016
1017 write(EMPTY_SET)
1018 self.memoize(obj)
1019
1020 it = iter(obj)
1021 while True:
1022 batch = list(islice(it, self._BATCHSIZE))
1023 n = len(batch)
1024 if n > 0:
1025 write(MARK)
1026 for item in batch:
1027 save(item)
1028 write(ADDITEMS)
1029 if n < self._BATCHSIZE:
1030 return
1031 dispatch[set] = save_set
1032
1033 def save_frozenset(self, obj):
1034 save = self.save
1035 write = self.write
1036
1037 if self.proto < 4:
1038 self.save_reduce(frozenset, (list(obj),), obj=obj)
1039 return
1040
1041 write(MARK)
1042 for item in obj:
1043 save(item)
1044
1045 if id(obj) in self.memo:
1046 # If the object is already in the memo, this means it is
1047 # recursive. In this case, throw away everything we put on the
1048 # stack, and fetch the object back from the memo.
1049 write(POP_MARK + self.get(self.memo[id(obj)][0]))
1050 return
1051
1052 write(FROZENSET)
1053 self.memoize(obj)
1054 dispatch[frozenset] = save_frozenset
1055
1056 def save_global(self, obj, name=None):
1057 write = self.write
1058 memo = self.memo
1059
1060 if name is None:
1061 name = getattr(obj, '__qualname__', None)
1062 if name is None:
1063 name = obj.__name__
1064
1065 module_name = whichmodule(obj, name)
1066 try:
1067 __import__(module_name, level=0)
1068 module = sys.modules[module_name]
1069 obj2, parent = _getattribute(module, name)
1070 except (ImportError, KeyError, AttributeError):
1071 raise PicklingError(
1072 "Can't pickle %r: it's not found as %s.%s" %
1073 (obj, module_name, name)) from None
1074 else:
1075 if obj2 is not obj:
1076 raise PicklingError(
1077 "Can't pickle %r: it's not the same object as %s.%s" %
1078 (obj, module_name, name))
1079
1080 if self.proto >= 2:
1081 code = _extension_registry.get((module_name, name))
1082 if code:
1083 assert code > 0
1084 if code <= 0xff:
1085 write(EXT1 + pack("<B", code))
1086 elif code <= 0xffff:
1087 write(EXT2 + pack("<H", code))
1088 else:
1089 write(EXT4 + pack("<i", code))
1090 return
1091 lastname = name.rpartition('.')[2]
1092 if parent is module:
1093 name = lastname
1094 # Non-ASCII identifiers are supported only with protocols >= 3.
1095 if self.proto >= 4:
1096 self.save(module_name)
1097 self.save(name)
1098 write(STACK_GLOBAL)
1099 elif parent is not module:
1100 self.save_reduce(getattr, (parent, lastname))
1101 elif self.proto >= 3:
1102 write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
1103 bytes(name, "utf-8") + b'\n')
1104 else:
1105 if self.fix_imports:
1106 r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
1107 r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
1108 if (module_name, name) in r_name_mapping:
1109 module_name, name = r_name_mapping[(module_name, name)]
1110 elif module_name in r_import_mapping:
1111 module_name = r_import_mapping[module_name]
1112 try:
1113 write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
1114 bytes(name, "ascii") + b'\n')
1115 except UnicodeEncodeError:
1116 raise PicklingError(
1117 "can't pickle global identifier '%s.%s' using "
1118 "pickle protocol %i" % (module, name, self.proto)) from None
1119
1120 self.memoize(obj)
1121
1122 def save_type(self, obj):
1123 if obj is type(None):
1124 return self.save_reduce(type, (None,), obj=obj)
1125 elif obj is type(NotImplemented):
1126 return self.save_reduce(type, (NotImplemented,), obj=obj)
1127 elif obj is type(...):
1128 return self.save_reduce(type, (...,), obj=obj)
1129 return self.save_global(obj)
1130
1131 dispatch[FunctionType] = save_global
1132 dispatch[type] = save_type
1133
1134
1135 # Unpickling machinery
1136
1137 class ESC[4;38;5;81m_Unpickler:
1138
1139 def __init__(self, file, *, fix_imports=True,
1140 encoding="ASCII", errors="strict", buffers=None):
1141 """This takes a binary file for reading a pickle data stream.
1142
1143 The protocol version of the pickle is detected automatically, so
1144 no proto argument is needed.
1145
1146 The argument *file* must have two methods, a read() method that
1147 takes an integer argument, and a readline() method that requires
1148 no arguments. Both methods should return bytes. Thus *file*
1149 can be a binary file object opened for reading, an io.BytesIO
1150 object, or any other custom object that meets this interface.
1151
1152 The file-like object must have two methods, a read() method
1153 that takes an integer argument, and a readline() method that
1154 requires no arguments. Both methods should return bytes.
1155 Thus file-like object can be a binary file object opened for
1156 reading, a BytesIO object, or any other custom object that
1157 meets this interface.
1158
1159 If *buffers* is not None, it should be an iterable of buffer-enabled
1160 objects that is consumed each time the pickle stream references
1161 an out-of-band buffer view. Such buffers have been given in order
1162 to the *buffer_callback* of a Pickler object.
1163
1164 If *buffers* is None (the default), then the buffers are taken
1165 from the pickle stream, assuming they are serialized there.
1166 It is an error for *buffers* to be None if the pickle stream
1167 was produced with a non-None *buffer_callback*.
1168
1169 Other optional arguments are *fix_imports*, *encoding* and
1170 *errors*, which are used to control compatibility support for
1171 pickle stream generated by Python 2. If *fix_imports* is True,
1172 pickle will try to map the old Python 2 names to the new names
1173 used in Python 3. The *encoding* and *errors* tell pickle how
1174 to decode 8-bit string instances pickled by Python 2; these
1175 default to 'ASCII' and 'strict', respectively. *encoding* can be
1176 'bytes' to read these 8-bit string instances as bytes objects.
1177 """
1178 self._buffers = iter(buffers) if buffers is not None else None
1179 self._file_readline = file.readline
1180 self._file_read = file.read
1181 self.memo = {}
1182 self.encoding = encoding
1183 self.errors = errors
1184 self.proto = 0
1185 self.fix_imports = fix_imports
1186
1187 def load(self):
1188 """Read a pickled object representation from the open file.
1189
1190 Return the reconstituted object hierarchy specified in the file.
1191 """
1192 # Check whether Unpickler was initialized correctly. This is
1193 # only needed to mimic the behavior of _pickle.Unpickler.dump().
1194 if not hasattr(self, "_file_read"):
1195 raise UnpicklingError("Unpickler.__init__() was not called by "
1196 "%s.__init__()" % (self.__class__.__name__,))
1197 self._unframer = _Unframer(self._file_read, self._file_readline)
1198 self.read = self._unframer.read
1199 self.readinto = self._unframer.readinto
1200 self.readline = self._unframer.readline
1201 self.metastack = []
1202 self.stack = []
1203 self.append = self.stack.append
1204 self.proto = 0
1205 read = self.read
1206 dispatch = self.dispatch
1207 try:
1208 while True:
1209 key = read(1)
1210 if not key:
1211 raise EOFError
1212 assert isinstance(key, bytes_types)
1213 dispatch[key[0]](self)
1214 except _Stop as stopinst:
1215 return stopinst.value
1216
1217 # Return a list of items pushed in the stack after last MARK instruction.
1218 def pop_mark(self):
1219 items = self.stack
1220 self.stack = self.metastack.pop()
1221 self.append = self.stack.append
1222 return items
1223
1224 def persistent_load(self, pid):
1225 raise UnpicklingError("unsupported persistent id encountered")
1226
1227 dispatch = {}
1228
1229 def load_proto(self):
1230 proto = self.read(1)[0]
1231 if not 0 <= proto <= HIGHEST_PROTOCOL:
1232 raise ValueError("unsupported pickle protocol: %d" % proto)
1233 self.proto = proto
1234 dispatch[PROTO[0]] = load_proto
1235
1236 def load_frame(self):
1237 frame_size, = unpack('<Q', self.read(8))
1238 if frame_size > sys.maxsize:
1239 raise ValueError("frame size > sys.maxsize: %d" % frame_size)
1240 self._unframer.load_frame(frame_size)
1241 dispatch[FRAME[0]] = load_frame
1242
1243 def load_persid(self):
1244 try:
1245 pid = self.readline()[:-1].decode("ascii")
1246 except UnicodeDecodeError:
1247 raise UnpicklingError(
1248 "persistent IDs in protocol 0 must be ASCII strings")
1249 self.append(self.persistent_load(pid))
1250 dispatch[PERSID[0]] = load_persid
1251
1252 def load_binpersid(self):
1253 pid = self.stack.pop()
1254 self.append(self.persistent_load(pid))
1255 dispatch[BINPERSID[0]] = load_binpersid
1256
1257 def load_none(self):
1258 self.append(None)
1259 dispatch[NONE[0]] = load_none
1260
1261 def load_false(self):
1262 self.append(False)
1263 dispatch[NEWFALSE[0]] = load_false
1264
1265 def load_true(self):
1266 self.append(True)
1267 dispatch[NEWTRUE[0]] = load_true
1268
1269 def load_int(self):
1270 data = self.readline()
1271 if data == FALSE[1:]:
1272 val = False
1273 elif data == TRUE[1:]:
1274 val = True
1275 else:
1276 val = int(data, 0)
1277 self.append(val)
1278 dispatch[INT[0]] = load_int
1279
1280 def load_binint(self):
1281 self.append(unpack('<i', self.read(4))[0])
1282 dispatch[BININT[0]] = load_binint
1283
1284 def load_binint1(self):
1285 self.append(self.read(1)[0])
1286 dispatch[BININT1[0]] = load_binint1
1287
1288 def load_binint2(self):
1289 self.append(unpack('<H', self.read(2))[0])
1290 dispatch[BININT2[0]] = load_binint2
1291
1292 def load_long(self):
1293 val = self.readline()[:-1]
1294 if val and val[-1] == b'L'[0]:
1295 val = val[:-1]
1296 self.append(int(val, 0))
1297 dispatch[LONG[0]] = load_long
1298
1299 def load_long1(self):
1300 n = self.read(1)[0]
1301 data = self.read(n)
1302 self.append(decode_long(data))
1303 dispatch[LONG1[0]] = load_long1
1304
1305 def load_long4(self):
1306 n, = unpack('<i', self.read(4))
1307 if n < 0:
1308 # Corrupt or hostile pickle -- we never write one like this
1309 raise UnpicklingError("LONG pickle has negative byte count")
1310 data = self.read(n)
1311 self.append(decode_long(data))
1312 dispatch[LONG4[0]] = load_long4
1313
1314 def load_float(self):
1315 self.append(float(self.readline()[:-1]))
1316 dispatch[FLOAT[0]] = load_float
1317
1318 def load_binfloat(self):
1319 self.append(unpack('>d', self.read(8))[0])
1320 dispatch[BINFLOAT[0]] = load_binfloat
1321
1322 def _decode_string(self, value):
1323 # Used to allow strings from Python 2 to be decoded either as
1324 # bytes or Unicode strings. This should be used only with the
1325 # STRING, BINSTRING and SHORT_BINSTRING opcodes.
1326 if self.encoding == "bytes":
1327 return value
1328 else:
1329 return value.decode(self.encoding, self.errors)
1330
1331 def load_string(self):
1332 data = self.readline()[:-1]
1333 # Strip outermost quotes
1334 if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
1335 data = data[1:-1]
1336 else:
1337 raise UnpicklingError("the STRING opcode argument must be quoted")
1338 self.append(self._decode_string(codecs.escape_decode(data)[0]))
1339 dispatch[STRING[0]] = load_string
1340
1341 def load_binstring(self):
1342 # Deprecated BINSTRING uses signed 32-bit length
1343 len, = unpack('<i', self.read(4))
1344 if len < 0:
1345 raise UnpicklingError("BINSTRING pickle has negative byte count")
1346 data = self.read(len)
1347 self.append(self._decode_string(data))
1348 dispatch[BINSTRING[0]] = load_binstring
1349
1350 def load_binbytes(self):
1351 len, = unpack('<I', self.read(4))
1352 if len > maxsize:
1353 raise UnpicklingError("BINBYTES exceeds system's maximum size "
1354 "of %d bytes" % maxsize)
1355 self.append(self.read(len))
1356 dispatch[BINBYTES[0]] = load_binbytes
1357
1358 def load_unicode(self):
1359 self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
1360 dispatch[UNICODE[0]] = load_unicode
1361
1362 def load_binunicode(self):
1363 len, = unpack('<I', self.read(4))
1364 if len > maxsize:
1365 raise UnpicklingError("BINUNICODE exceeds system's maximum size "
1366 "of %d bytes" % maxsize)
1367 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1368 dispatch[BINUNICODE[0]] = load_binunicode
1369
1370 def load_binunicode8(self):
1371 len, = unpack('<Q', self.read(8))
1372 if len > maxsize:
1373 raise UnpicklingError("BINUNICODE8 exceeds system's maximum size "
1374 "of %d bytes" % maxsize)
1375 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1376 dispatch[BINUNICODE8[0]] = load_binunicode8
1377
1378 def load_binbytes8(self):
1379 len, = unpack('<Q', self.read(8))
1380 if len > maxsize:
1381 raise UnpicklingError("BINBYTES8 exceeds system's maximum size "
1382 "of %d bytes" % maxsize)
1383 self.append(self.read(len))
1384 dispatch[BINBYTES8[0]] = load_binbytes8
1385
1386 def load_bytearray8(self):
1387 len, = unpack('<Q', self.read(8))
1388 if len > maxsize:
1389 raise UnpicklingError("BYTEARRAY8 exceeds system's maximum size "
1390 "of %d bytes" % maxsize)
1391 b = bytearray(len)
1392 self.readinto(b)
1393 self.append(b)
1394 dispatch[BYTEARRAY8[0]] = load_bytearray8
1395
1396 def load_next_buffer(self):
1397 if self._buffers is None:
1398 raise UnpicklingError("pickle stream refers to out-of-band data "
1399 "but no *buffers* argument was given")
1400 try:
1401 buf = next(self._buffers)
1402 except StopIteration:
1403 raise UnpicklingError("not enough out-of-band buffers")
1404 self.append(buf)
1405 dispatch[NEXT_BUFFER[0]] = load_next_buffer
1406
1407 def load_readonly_buffer(self):
1408 buf = self.stack[-1]
1409 with memoryview(buf) as m:
1410 if not m.readonly:
1411 self.stack[-1] = m.toreadonly()
1412 dispatch[READONLY_BUFFER[0]] = load_readonly_buffer
1413
1414 def load_short_binstring(self):
1415 len = self.read(1)[0]
1416 data = self.read(len)
1417 self.append(self._decode_string(data))
1418 dispatch[SHORT_BINSTRING[0]] = load_short_binstring
1419
1420 def load_short_binbytes(self):
1421 len = self.read(1)[0]
1422 self.append(self.read(len))
1423 dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
1424
1425 def load_short_binunicode(self):
1426 len = self.read(1)[0]
1427 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1428 dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode
1429
1430 def load_tuple(self):
1431 items = self.pop_mark()
1432 self.append(tuple(items))
1433 dispatch[TUPLE[0]] = load_tuple
1434
1435 def load_empty_tuple(self):
1436 self.append(())
1437 dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
1438
1439 def load_tuple1(self):
1440 self.stack[-1] = (self.stack[-1],)
1441 dispatch[TUPLE1[0]] = load_tuple1
1442
1443 def load_tuple2(self):
1444 self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
1445 dispatch[TUPLE2[0]] = load_tuple2
1446
1447 def load_tuple3(self):
1448 self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
1449 dispatch[TUPLE3[0]] = load_tuple3
1450
1451 def load_empty_list(self):
1452 self.append([])
1453 dispatch[EMPTY_LIST[0]] = load_empty_list
1454
1455 def load_empty_dictionary(self):
1456 self.append({})
1457 dispatch[EMPTY_DICT[0]] = load_empty_dictionary
1458
1459 def load_empty_set(self):
1460 self.append(set())
1461 dispatch[EMPTY_SET[0]] = load_empty_set
1462
1463 def load_frozenset(self):
1464 items = self.pop_mark()
1465 self.append(frozenset(items))
1466 dispatch[FROZENSET[0]] = load_frozenset
1467
1468 def load_list(self):
1469 items = self.pop_mark()
1470 self.append(items)
1471 dispatch[LIST[0]] = load_list
1472
1473 def load_dict(self):
1474 items = self.pop_mark()
1475 d = {items[i]: items[i+1]
1476 for i in range(0, len(items), 2)}
1477 self.append(d)
1478 dispatch[DICT[0]] = load_dict
1479
1480 # INST and OBJ differ only in how they get a class object. It's not
1481 # only sensible to do the rest in a common routine, the two routines
1482 # previously diverged and grew different bugs.
1483 # klass is the class to instantiate, and k points to the topmost mark
1484 # object, following which are the arguments for klass.__init__.
1485 def _instantiate(self, klass, args):
1486 if (args or not isinstance(klass, type) or
1487 hasattr(klass, "__getinitargs__")):
1488 try:
1489 value = klass(*args)
1490 except TypeError as err:
1491 raise TypeError("in constructor for %s: %s" %
1492 (klass.__name__, str(err)), sys.exc_info()[2])
1493 else:
1494 value = klass.__new__(klass)
1495 self.append(value)
1496
1497 def load_inst(self):
1498 module = self.readline()[:-1].decode("ascii")
1499 name = self.readline()[:-1].decode("ascii")
1500 klass = self.find_class(module, name)
1501 self._instantiate(klass, self.pop_mark())
1502 dispatch[INST[0]] = load_inst
1503
1504 def load_obj(self):
1505 # Stack is ... markobject classobject arg1 arg2 ...
1506 args = self.pop_mark()
1507 cls = args.pop(0)
1508 self._instantiate(cls, args)
1509 dispatch[OBJ[0]] = load_obj
1510
1511 def load_newobj(self):
1512 args = self.stack.pop()
1513 cls = self.stack.pop()
1514 obj = cls.__new__(cls, *args)
1515 self.append(obj)
1516 dispatch[NEWOBJ[0]] = load_newobj
1517
1518 def load_newobj_ex(self):
1519 kwargs = self.stack.pop()
1520 args = self.stack.pop()
1521 cls = self.stack.pop()
1522 obj = cls.__new__(cls, *args, **kwargs)
1523 self.append(obj)
1524 dispatch[NEWOBJ_EX[0]] = load_newobj_ex
1525
1526 def load_global(self):
1527 module = self.readline()[:-1].decode("utf-8")
1528 name = self.readline()[:-1].decode("utf-8")
1529 klass = self.find_class(module, name)
1530 self.append(klass)
1531 dispatch[GLOBAL[0]] = load_global
1532
1533 def load_stack_global(self):
1534 name = self.stack.pop()
1535 module = self.stack.pop()
1536 if type(name) is not str or type(module) is not str:
1537 raise UnpicklingError("STACK_GLOBAL requires str")
1538 self.append(self.find_class(module, name))
1539 dispatch[STACK_GLOBAL[0]] = load_stack_global
1540
1541 def load_ext1(self):
1542 code = self.read(1)[0]
1543 self.get_extension(code)
1544 dispatch[EXT1[0]] = load_ext1
1545
1546 def load_ext2(self):
1547 code, = unpack('<H', self.read(2))
1548 self.get_extension(code)
1549 dispatch[EXT2[0]] = load_ext2
1550
1551 def load_ext4(self):
1552 code, = unpack('<i', self.read(4))
1553 self.get_extension(code)
1554 dispatch[EXT4[0]] = load_ext4
1555
1556 def get_extension(self, code):
1557 nil = []
1558 obj = _extension_cache.get(code, nil)
1559 if obj is not nil:
1560 self.append(obj)
1561 return
1562 key = _inverted_registry.get(code)
1563 if not key:
1564 if code <= 0: # note that 0 is forbidden
1565 # Corrupt or hostile pickle.
1566 raise UnpicklingError("EXT specifies code <= 0")
1567 raise ValueError("unregistered extension code %d" % code)
1568 obj = self.find_class(*key)
1569 _extension_cache[code] = obj
1570 self.append(obj)
1571
1572 def find_class(self, module, name):
1573 # Subclasses may override this.
1574 sys.audit('pickle.find_class', module, name)
1575 if self.proto < 3 and self.fix_imports:
1576 if (module, name) in _compat_pickle.NAME_MAPPING:
1577 module, name = _compat_pickle.NAME_MAPPING[(module, name)]
1578 elif module in _compat_pickle.IMPORT_MAPPING:
1579 module = _compat_pickle.IMPORT_MAPPING[module]
1580 __import__(module, level=0)
1581 if self.proto >= 4:
1582 return _getattribute(sys.modules[module], name)[0]
1583 else:
1584 return getattr(sys.modules[module], name)
1585
1586 def load_reduce(self):
1587 stack = self.stack
1588 args = stack.pop()
1589 func = stack[-1]
1590 stack[-1] = func(*args)
1591 dispatch[REDUCE[0]] = load_reduce
1592
1593 def load_pop(self):
1594 if self.stack:
1595 del self.stack[-1]
1596 else:
1597 self.pop_mark()
1598 dispatch[POP[0]] = load_pop
1599
1600 def load_pop_mark(self):
1601 self.pop_mark()
1602 dispatch[POP_MARK[0]] = load_pop_mark
1603
1604 def load_dup(self):
1605 self.append(self.stack[-1])
1606 dispatch[DUP[0]] = load_dup
1607
1608 def load_get(self):
1609 i = int(self.readline()[:-1])
1610 try:
1611 self.append(self.memo[i])
1612 except KeyError:
1613 msg = f'Memo value not found at index {i}'
1614 raise UnpicklingError(msg) from None
1615 dispatch[GET[0]] = load_get
1616
1617 def load_binget(self):
1618 i = self.read(1)[0]
1619 try:
1620 self.append(self.memo[i])
1621 except KeyError as exc:
1622 msg = f'Memo value not found at index {i}'
1623 raise UnpicklingError(msg) from None
1624 dispatch[BINGET[0]] = load_binget
1625
1626 def load_long_binget(self):
1627 i, = unpack('<I', self.read(4))
1628 try:
1629 self.append(self.memo[i])
1630 except KeyError as exc:
1631 msg = f'Memo value not found at index {i}'
1632 raise UnpicklingError(msg) from None
1633 dispatch[LONG_BINGET[0]] = load_long_binget
1634
1635 def load_put(self):
1636 i = int(self.readline()[:-1])
1637 if i < 0:
1638 raise ValueError("negative PUT argument")
1639 self.memo[i] = self.stack[-1]
1640 dispatch[PUT[0]] = load_put
1641
1642 def load_binput(self):
1643 i = self.read(1)[0]
1644 if i < 0:
1645 raise ValueError("negative BINPUT argument")
1646 self.memo[i] = self.stack[-1]
1647 dispatch[BINPUT[0]] = load_binput
1648
1649 def load_long_binput(self):
1650 i, = unpack('<I', self.read(4))
1651 if i > maxsize:
1652 raise ValueError("negative LONG_BINPUT argument")
1653 self.memo[i] = self.stack[-1]
1654 dispatch[LONG_BINPUT[0]] = load_long_binput
1655
1656 def load_memoize(self):
1657 memo = self.memo
1658 memo[len(memo)] = self.stack[-1]
1659 dispatch[MEMOIZE[0]] = load_memoize
1660
1661 def load_append(self):
1662 stack = self.stack
1663 value = stack.pop()
1664 list = stack[-1]
1665 list.append(value)
1666 dispatch[APPEND[0]] = load_append
1667
1668 def load_appends(self):
1669 items = self.pop_mark()
1670 list_obj = self.stack[-1]
1671 try:
1672 extend = list_obj.extend
1673 except AttributeError:
1674 pass
1675 else:
1676 extend(items)
1677 return
1678 # Even if the PEP 307 requires extend() and append() methods,
1679 # fall back on append() if the object has no extend() method
1680 # for backward compatibility.
1681 append = list_obj.append
1682 for item in items:
1683 append(item)
1684 dispatch[APPENDS[0]] = load_appends
1685
1686 def load_setitem(self):
1687 stack = self.stack
1688 value = stack.pop()
1689 key = stack.pop()
1690 dict = stack[-1]
1691 dict[key] = value
1692 dispatch[SETITEM[0]] = load_setitem
1693
1694 def load_setitems(self):
1695 items = self.pop_mark()
1696 dict = self.stack[-1]
1697 for i in range(0, len(items), 2):
1698 dict[items[i]] = items[i + 1]
1699 dispatch[SETITEMS[0]] = load_setitems
1700
1701 def load_additems(self):
1702 items = self.pop_mark()
1703 set_obj = self.stack[-1]
1704 if isinstance(set_obj, set):
1705 set_obj.update(items)
1706 else:
1707 add = set_obj.add
1708 for item in items:
1709 add(item)
1710 dispatch[ADDITEMS[0]] = load_additems
1711
1712 def load_build(self):
1713 stack = self.stack
1714 state = stack.pop()
1715 inst = stack[-1]
1716 setstate = getattr(inst, "__setstate__", None)
1717 if setstate is not None:
1718 setstate(state)
1719 return
1720 slotstate = None
1721 if isinstance(state, tuple) and len(state) == 2:
1722 state, slotstate = state
1723 if state:
1724 inst_dict = inst.__dict__
1725 intern = sys.intern
1726 for k, v in state.items():
1727 if type(k) is str:
1728 inst_dict[intern(k)] = v
1729 else:
1730 inst_dict[k] = v
1731 if slotstate:
1732 for k, v in slotstate.items():
1733 setattr(inst, k, v)
1734 dispatch[BUILD[0]] = load_build
1735
1736 def load_mark(self):
1737 self.metastack.append(self.stack)
1738 self.stack = []
1739 self.append = self.stack.append
1740 dispatch[MARK[0]] = load_mark
1741
1742 def load_stop(self):
1743 value = self.stack.pop()
1744 raise _Stop(value)
1745 dispatch[STOP[0]] = load_stop
1746
1747
1748 # Shorthands
1749
1750 def _dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None):
1751 _Pickler(file, protocol, fix_imports=fix_imports,
1752 buffer_callback=buffer_callback).dump(obj)
1753
1754 def _dumps(obj, protocol=None, *, fix_imports=True, buffer_callback=None):
1755 f = io.BytesIO()
1756 _Pickler(f, protocol, fix_imports=fix_imports,
1757 buffer_callback=buffer_callback).dump(obj)
1758 res = f.getvalue()
1759 assert isinstance(res, bytes_types)
1760 return res
1761
1762 def _load(file, *, fix_imports=True, encoding="ASCII", errors="strict",
1763 buffers=None):
1764 return _Unpickler(file, fix_imports=fix_imports, buffers=buffers,
1765 encoding=encoding, errors=errors).load()
1766
1767 def _loads(s, /, *, fix_imports=True, encoding="ASCII", errors="strict",
1768 buffers=None):
1769 if isinstance(s, str):
1770 raise TypeError("Can't load pickle from unicode string")
1771 file = io.BytesIO(s)
1772 return _Unpickler(file, fix_imports=fix_imports, buffers=buffers,
1773 encoding=encoding, errors=errors).load()
1774
1775 # Use the faster _pickle if possible
1776 try:
1777 from _pickle import (
1778 PickleError,
1779 PicklingError,
1780 UnpicklingError,
1781 Pickler,
1782 Unpickler,
1783 dump,
1784 dumps,
1785 load,
1786 loads
1787 )
1788 except ImportError:
1789 Pickler, Unpickler = _Pickler, _Unpickler
1790 dump, dumps, load, loads = _dump, _dumps, _load, _loads
1791
1792 # Doctest
1793 def _test():
1794 import doctest
1795 return doctest.testmod()
1796
1797 if __name__ == "__main__":
1798 import argparse
1799 parser = argparse.ArgumentParser(
1800 description='display contents of the pickle files')
1801 parser.add_argument(
1802 'pickle_file', type=argparse.FileType('br'),
1803 nargs='*', help='the pickle file')
1804 parser.add_argument(
1805 '-t', '--test', action='store_true',
1806 help='run self-test suite')
1807 parser.add_argument(
1808 '-v', action='store_true',
1809 help='run verbosely; only affects self-test run')
1810 args = parser.parse_args()
1811 if args.test:
1812 _test()
1813 else:
1814 if not args.pickle_file:
1815 parser.print_help()
1816 else:
1817 import pprint
1818 for f in args.pickle_file:
1819 obj = load(f)
1820 pprint.pprint(obj)