PC World 2002 May

home *** CD-ROM | disk | FTP | other *** search

/ PC World 2002 May / PCWorld_2002-05_cd.bin / Software / TemaCD / activepython / ActivePython-2.1.1.msi / Python21_Lib_pickle.py < prev next >

Wrap

Python Source | 2001-07-26 | 26.9 KB | 987 lines

"""Create portable serialized representations of Python objects. See module cPickle for a (much) faster implementation. See module copy_reg for a mechanism for registering custom picklers. Classes: Pickler Unpickler Functions: dump(object, file) dumps(object) -> string load(file) -> object loads(string) -> object Misc variables: __version__ format_version compatible_formats """ __version__ = "$Revision: 1.48 $" # Code version from types import * from copy_reg import dispatch_table, safe_constructors import marshal import sys import struct import re __all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler", "Unpickler", "dump", "dumps", "load", "loads"] format_version = "1.3" # File format version we write compatible_formats = ["1.0", "1.1", "1.2"] # Old format versions we can read mdumps = marshal.dumps mloads = marshal.loads class PickleError(Exception): pass class PicklingError(PickleError): pass class UnpicklingError(PickleError): pass class _Stop(Exception): def __init__(self, value): self.value = value try: from org.python.core import PyStringMap except ImportError: PyStringMap = None MARK = '(' STOP = '.' POP = '0' POP_MARK = '1' DUP = '2' FLOAT = 'F' INT = 'I' BININT = 'J' BININT1 = 'K' LONG = 'L' BININT2 = 'M' NONE = 'N' PERSID = 'P' BINPERSID = 'Q' REDUCE = 'R' STRING = 'S' BINSTRING = 'T' SHORT_BINSTRING = 'U' UNICODE = 'V' BINUNICODE = 'X' APPEND = 'a' BUILD = 'b' GLOBAL = 'c' DICT = 'd' EMPTY_DICT = '}' APPENDS = 'e' GET = 'g' BINGET = 'h' INST = 'i' LONG_BINGET = 'j' LIST = 'l' EMPTY_LIST = ']' OBJ = 'o' PUT = 'p' BINPUT = 'q' LONG_BINPUT = 'r' SETITEM = 's' TUPLE = 't' EMPTY_TUPLE = ')' SETITEMS = 'u' BINFLOAT = 'G' __all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)]) class Pickler: def __init__(self, file, bin = 0): self.write = file.write self.memo = {} self.bin = bin def dump(self, object): self.save(object) self.write(STOP) def put(self, i): if self.bin: s = mdumps(i)[1:] if i < 256: return BINPUT + s[0] return LONG_BINPUT + s return PUT + `i` + '\n' def get(self, i): if self.bin: s = mdumps(i)[1:] if i < 256: return BINGET + s[0] return LONG_BINGET + s return GET + `i` + '\n' def save(self, object, pers_save = 0): memo = self.memo if not pers_save: pid = self.persistent_id(object) if pid is not None: self.save_pers(pid) return d = id(object) t = type(object) if (t is TupleType) and (len(object) == 0): if self.bin: self.save_empty_tuple(object) else: self.save_tuple(object) return if memo.has_key(d): self.write(self.get(memo[d][0])) return try: f = self.dispatch[t] except KeyError: pid = self.inst_persistent_id(object) if pid is not None: self.save_pers(pid) return try: reduce = dispatch_table[t] except KeyError: try: reduce = object.__reduce__ except AttributeError: raise PicklingError, \ "can't pickle %s object: %s" % (`t.__name__`, `object`) else: tup = reduce() else: tup = reduce(object) if type(tup) is StringType: self.save_global(object, tup) return if type(tup) is not TupleType: raise PicklingError, "Value returned by %s must be a " \ "tuple" % reduce l = len(tup) if (l != 2) and (l != 3): raise PicklingError, "tuple returned by %s must contain " \ "only two or three elements" % reduce callable = tup[0] arg_tup = tup[1] if l > 2: state = tup[2] else: state = None if type(arg_tup) is not TupleType and arg_tup is not None: raise PicklingError, "Second element of tuple returned " \ "by %s must be a tuple" % reduce self.save_reduce(callable, arg_tup, state) memo_len = len(memo) self.write(self.put(memo_len)) memo[d] = (memo_len, object) return f(self, object) def persistent_id(self, object): return None def inst_persistent_id(self, object): return None def save_pers(self, pid): if not self.bin: self.write(PERSID + str(pid) + '\n') else: self.save(pid, 1) self.write(BINPERSID) def save_reduce(self, callable, arg_tup, state = None): write = self.write save = self.save save(callable) save(arg_tup) write(REDUCE) if state is not None: save(state) write(BUILD) dispatch = {} def save_none(self, object): self.write(NONE) dispatch[NoneType] = save_none def save_int(self, object): if self.bin: # If the int is small enough to fit in a signed 4-byte 2's-comp # format, we can store it more efficiently than the general # case. high_bits = object >> 31 # note that Python shift sign-extends if high_bits == 0 or high_bits == -1: # All high bits are copies of bit 2**31, so the value # fits in a 4-byte signed int. i = mdumps(object)[1:] assert len(i) == 4 if i[-2:] == '\000\000': # fits in 2-byte unsigned int if i[-3] == '\000': # fits in 1-byte unsigned int self.write(BININT1 + i[0]) else: self.write(BININT2 + i[:2]) else: self.write(BININT + i) return # Text pickle, or int too big to fit in signed 4-byte format. self.write(INT + `object` + '\n') dispatch[IntType] = save_int def save_long(self, object): self.write(LONG + `object` + '\n') dispatch[LongType] = save_long def save_float(self, object, pack=struct.pack): if self.bin: self.write(BINFLOAT + pack('>d', object)) else: self.write(FLOAT + `object` + '\n') dispatch[FloatType] = save_float def save_string(self, object): d = id(object) memo = self.memo if self.bin: l = len(object) s = mdumps(l)[1:] if l < 256: self.write(SHORT_BINSTRING + s[0] + object) else: self.write(BINSTRING + s + object) else: self.write(STRING + `object` + '\n') memo_len = len(memo) self.write(self.put(memo_len)) memo[d] = (memo_len, object) dispatch[StringType] = save_string def save_unicode(self, object): d = id(object) memo = self.memo if self.bin: encoding = object.encode('utf-8') l = len(encoding) s = mdumps(l)[1:] self.write(BINUNICODE + s + encoding) else: object = object.replace(u"\\", u"\\u005c") object = object.replace(u"\n", u"\\u000a") self.write(UNICODE + object.encode('raw-unicode-escape') + '\n') memo_len = len(memo) self.write(self.put(memo_len)) memo[d] = (memo_len, object) dispatch[UnicodeType] = save_unicode if StringType == UnicodeType: # This is true for Jython def save_string(self, object): d = id(object) memo = self.memo unicode = object.isunicode() if self.bin: if unicode: object = object.encode("utf-8") l = len(object) s = mdumps(l)[1:] if l < 256 and not unicode: self.write(SHORT_BINSTRING + s[0] + object) else: if unicode: self.write(BINUNICODE + s + object) else: self.write(BINSTRING + s + object) else: if unicode: object = object.replace(u"\\", u"\\u005c") object = object.replace(u"\n", u"\\u000a") object = object.encode('raw-unicode-escape') self.write(UNICODE + object + '\n') else: self.write(STRING + `object` + '\n') memo_len = len(memo) self.write(self.put(memo_len)) memo[d] = (memo_len, object) dispatch[StringType] = save_string def save_tuple(self, object): write = self.write save = self.save memo = self.memo d = id(object) write(MARK) for element in object: save(element) if len(object) and memo.has_key(d): if self.bin: write(POP_MARK + self.get(memo[d][0])) return write(POP * (len(object) + 1) + self.get(memo[d][0])) return memo_len = len(memo) self.write(TUPLE + self.put(memo_len)) memo[d] = (memo_len, object) dispatch[TupleType] = save_tuple def save_empty_tuple(self, object): self.write(EMPTY_TUPLE) def save_list(self, object): d = id(object) write = self.write save = self.save memo = self.memo if self.bin: write(EMPTY_LIST) else: write(MARK + LIST) memo_len = len(memo) write(self.put(memo_len)) memo[d] = (memo_len, object) using_appends = (self.bin and (len(object) > 1)) if using_appends: write(MARK) for element in object: save(element) if not using_appends: write(APPEND) if using_appends: write(APPENDS) dispatch[ListType] = save_list def save_dict(self, object): d = id(object) write = self.write save = self.save memo = self.memo if self.bin: write(EMPTY_DICT) else: write(MARK + DICT) memo_len = len(memo) self.write(self.put(memo_len)) memo[d] = (memo_len, object) using_setitems = (self.bin and (len(object) > 1)) if using_setitems: write(MARK) items = object.items() for key, value in items: save(key) save(value) if not using_setitems: write(SETITEM) if using_setitems: write(SETITEMS) dispatch[DictionaryType] = save_dict if not PyStringMap is None: dispatch[PyStringMap] = save_dict def save_inst(self, object): d = id(object) cls = object.__class__ memo = self.memo write = self.write save = self.save if hasattr(object, '__getinitargs__'): args = object.__getinitargs__() len(args) # XXX Assert it's a sequence _keep_alive(args, memo) else: args = () write(MARK) if self.bin: save(cls) for arg in args: save(arg) memo_len = len(memo) if self.bin: write(OBJ + self.put(memo_len)) else: write(INST + cls.__module__ + '\n' + cls.__name__ + '\n' + self.put(memo_len)) memo[d] = (memo_len, object) try: getstate = object.__getstate__ except AttributeError: stuff = object.__dict__ else: stuff = getstate() _keep_alive(stuff, memo) save(stuff) write(BUILD) dispatch[InstanceType] = save_inst def save_global(self, object, name = None): write = self.write memo = self.memo if name is None: name = object.__name__ try: module = object.__module__ except AttributeError: module = whichmodule(object, name) memo_len = len(memo) write(GLOBAL + module + '\n' + name + '\n' + self.put(memo_len)) memo[id(object)] = (memo_len, object) dispatch[ClassType] = save_global dispatch[FunctionType] = save_global dispatch[BuiltinFunctionType] = save_global def _keep_alive(x, memo): """Keeps a reference to the object x in the memo. Because we remember objects by their id, we have to assure that possibly temporary objects are kept alive by referencing them. We store a reference at the id of the memo, which should normally not be used unless someone tries to deepcopy the memo itself... """ try: memo[id(memo)].append(x) except KeyError: # aha, this is the first one :-) memo[id(memo)]=[x] classmap = {} # This is no longer used to find classes, but still for functions def whichmodule(cls, clsname): """Figure out the module in which a class occurs. Search sys.modules for the module. Cache in classmap. Return a module name. If the class cannot be found, return __main__. """ if classmap.has_key(cls): return classmap[cls] for name, module in sys.modules.items(): if name != '__main__' and \ hasattr(module, clsname) and \ getattr(module, clsname) is cls: break else: name = '__main__' classmap[cls] = name return name class Unpickler: def __init__(self, file): self.readline = file.readline self.read = file.read self.memo = {} def load(self): self.mark = ['spam'] # Any new unique object self.stack = [] self.append = self.stack.append read = self.read dispatch = self.dispatch try: while 1: key = read(1) dispatch[key](self) except _Stop, stopinst: return stopinst.value def marker(self): stack = self.stack mark = self.mark k = len(stack)-1 while stack[k] is not mark: k = k-1 return k dispatch = {} def load_eof(self): raise EOFError dispatch[''] = load_eof def load_persid(self): pid = self.readline()[:-1] self.append(self.persistent_load(pid)) dispatch[PERSID] = load_persid def load_binpersid(self): stack = self.stack pid = stack[-1] del stack[-1] self.append(self.persistent_load(pid)) dispatch[BINPERSID] = load_binpersid def load_none(self): self.append(None) dispatch[NONE] = load_none def load_int(self): self.append(int(self.readline()[:-1])) dispatch[INT] = load_int def load_binint(self): self.append(mloads('i' + self.read(4))) dispatch[BININT] = load_binint def load_binint1(self): self.append(mloads('i' + self.read(1) + '\000\000\000')) dispatch[BININT1] = load_binint1 def load_binint2(self): self.append(mloads('i' + self.read(2) + '\000\000')) dispatch[BININT2] = load_binint2 def load_long(self): self.append(long(self.readline()[:-1], 0)) dispatch[LONG] = load_long def load_float(self): self.append(float(self.readline()[:-1])) dispatch[FLOAT] = load_float def load_binfloat(self, unpack=struct.unpack): self.append(unpack('>d', self.read(8))[0]) dispatch[BINFLOAT] = load_binfloat def load_string(self): rep = self.readline()[:-1] if not self._is_string_secure(rep): raise ValueError, "insecure string pickle" self.append(eval(rep, {'__builtins__': {}})) # Let's be careful dispatch[STRING] = load_string def _is_string_secure(self, s): """Return true if s contains a string that is safe to eval The definition of secure string is based on the implementation in cPickle. s is secure as long as it only contains a quoted string and optional trailing whitespace. """ q = s[0] if q not in ("'", '"'): return 0 # find the closing quote offset = 1 i = None while 1: try: i = s.index(q, offset) except ValueError: # if there is an error the first time, there is no # close quote if offset == 1: return 0 if s[i-1] != '\\': break # check to see if this one is escaped nslash = 0 j = i - 1 while j >= offset and s[j] == '\\': j = j - 1 nslash = nslash + 1 if nslash % 2 == 0: break offset = i + 1 for c in s[i+1:]: if ord(c) > 32: return 0 return 1 def load_binstring(self): len = mloads('i' + self.read(4)) self.append(self.read(len)) dispatch[BINSTRING] = load_binstring def load_unicode(self): self.append(unicode(self.readline()[:-1],'raw-unicode-escape')) dispatch[UNICODE] = load_unicode def load_binunicode(self): len = mloads('i' + self.read(4)) self.append(unicode(self.read(len),'utf-8')) dispatch[BINUNICODE] = load_binunicode def load_short_binstring(self): len = mloads('i' + self.read(1) + '\000\000\000') self.append(self.read(len)) dispatch[SHORT_BINSTRING] = load_short_binstring def load_tuple(self): k = self.marker() self.stack[k:] = [tuple(self.stack[k+1:])] dispatch[TUPLE] = load_tuple def load_empty_tuple(self): self.stack.append(()) dispatch[EMPTY_TUPLE] = load_empty_tuple def load_empty_list(self): self.stack.append([]) dispatch[EMPTY_LIST] = load_empty_list def load_empty_dictionary(self): self.stack.append({}) dispatch[EMPTY_DICT] = load_empty_dictionary def load_list(self): k = self.marker() self.stack[k:] = [self.stack[k+1:]] dispatch[LIST] = load_list def load_dict(self): k = self.marker() d = {} items = self.stack[k+1:] for i in range(0, len(items), 2): key = items[i] value = items[i+1] d[key] = value self.stack[k:] = [d] dispatch[DICT] = load_dict def load_inst(self): k = self.marker() args = tuple(self.stack[k+1:]) del self.stack[k:] module = self.readline()[:-1] name = self.readline()[:-1] klass = self.find_class(module, name) instantiated = 0 if (not args and type(klass) is ClassType and not hasattr(klass, "__getinitargs__")): try: value = _EmptyClass() value.__class__ = klass instantiated = 1 except RuntimeError: # In restricted execution, assignment to inst.__class__ is # prohibited pass if not instantiated: try: value = apply(klass, args) except TypeError, err: raise TypeError, "in constructor for %s: %s" % ( klass.__name__, str(err)), sys.exc_info()[2] self.append(value) dispatch[INST] = load_inst def load_obj(self): stack = self.stack k = self.marker() klass = stack[k + 1] del stack[k + 1] args = tuple(stack[k + 1:]) del stack[k:] instantiated = 0 if (not args and type(klass) is ClassType and not hasattr(klass, "__getinitargs__")): try: value = _EmptyClass() value.__class__ = klass instantiated = 1 except RuntimeError: # In restricted execution, assignment to inst.__class__ is # prohibited pass if not instantiated: value = apply(klass, args) self.append(value) dispatch[OBJ] = load_obj def load_global(self): module = self.readline()[:-1] name = self.readline()[:-1] klass = self.find_class(module, name) self.append(klass) dispatch[GLOBAL] = load_global def find_class(self, module, name): try: __import__(module) mod = sys.modules[module] klass = getattr(mod, name) except (ImportError, KeyError, AttributeError): raise SystemError, \ "Failed to import class %s from module %s" % \ (name, module) return klass def load_reduce(self): stack = self.stack callable = stack[-2] arg_tup = stack[-1] del stack[-2:] if type(callable) is not ClassType: if not safe_constructors.has_key(callable): try: safe = callable.__safe_for_unpickling__ except AttributeError: safe = None if not safe: raise UnpicklingError, "%s is not safe for " \ "unpickling" % callable if arg_tup is None: value = callable.__basicnew__() else: value = apply(callable, arg_tup) self.append(value) dispatch[REDUCE] = load_reduce def load_pop(self): del self.stack[-1] dispatch[POP] = load_pop def load_pop_mark(self): k = self.marker() del self.stack[k:] dispatch[POP_MARK] = load_pop_mark def load_dup(self): self.append(self.stack[-1]) dispatch[DUP] = load_dup def load_get(self): self.append(self.memo[self.readline()[:-1]]) dispatch[GET] = load_get def load_binget(self): i = mloads('i' + self.read(1) + '\000\000\000') self.append(self.memo[`i`]) dispatch[BINGET] = load_binget def load_long_binget(self): i = mloads('i' + self.read(4)) self.append(self.memo[`i`]) dispatch[LONG_BINGET] = load_long_binget def load_put(self): self.memo[self.readline()[:-1]] = self.stack[-1] dispatch[PUT] = load_put def load_binput(self): i = mloads('i' + self.read(1) + '\000\000\000') self.memo[`i`] = self.stack[-1] dispatch[BINPUT] = load_binput def load_long_binput(self): i = mloads('i' + self.read(4)) self.memo[`i`] = self.stack[-1] dispatch[LONG_BINPUT] = load_long_binput def load_append(self): stack = self.stack value = stack[-1] del stack[-1] list = stack[-1] list.append(value) dispatch[APPEND] = load_append def load_appends(self): stack = self.stack mark = self.marker() list = stack[mark - 1] for i in range(mark + 1, len(stack)): list.append(stack[i]) del stack[mark:] dispatch[APPENDS] = load_appends def load_setitem(self): stack = self.stack value = stack[-1] key = stack[-2] del stack[-2:] dict = stack[-1] dict[key] = value dispatch[SETITEM] = load_setitem def load_setitems(self): stack = self.stack mark = self.marker() dict = stack[mark - 1] for i in range(mark + 1, len(stack), 2): dict[stack[i]] = stack[i + 1] del stack[mark:] dispatch[SETITEMS] = load_setitems def load_build(self): stack = self.stack value = stack[-1] del stack[-1] inst = stack[-1] try: setstate = inst.__setstate__ except AttributeError: try: inst.__dict__.update(value) except RuntimeError: # XXX In restricted execution, the instance's __dict__ is not # accessible. Use the old way of unpickling the instance # variables. This is a semantic different when unpickling in # restricted vs. unrestricted modes. for k, v in value.items(): setattr(inst, k, v) else: setstate(value) dispatch[BUILD] = load_build def load_mark(self): self.append(self.mark) dispatch[MARK] = load_mark def load_stop(self): value = self.stack[-1] del self.stack[-1] raise _Stop(value) dispatch[STOP] = load_stop # Helper class for load_inst/load_obj class _EmptyClass: pass # Shorthands from StringIO import StringIO def dump(object, file, bin = 0): Pickler(file, bin).dump(object) def dumps(object, bin = 0): file = StringIO() Pickler(file, bin).dump(object) return file.getvalue() def load(file): return Unpickler(file).load() def loads(str): file = StringIO(str) return Unpickler(file).load() # The rest is used for testing only class C: def __cmp__(self, other): return cmp(self.__dict__, other.__dict__) def test(): fn = 'out' c = C() c.foo = 1 c.bar = 2 x = [0, 1, 2, 3] y = ('abc', 'abc', c, c) x.append(y) x.append(y) x.append(5) f = open(fn, 'w') F = Pickler(f) F.dump(x) f.close() f = open(fn, 'r') U = Unpickler(f) x2 = U.load() print x print x2 print x == x2 print map(id, x) print map(id, x2) print F.memo print U.memo if __name__ == '__main__': test()