diff --git a/dill/_dill.py b/dill/_dill.py index 1c3caaed..31a09d6b 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -747,31 +747,31 @@ def _create_typemap(): else: _typemap = dict((v, k) for k, v in _reverse_typemap.iteritems()) -def _unmarshal(string): - return marshal.loads(string) - def _load_type(name): return _reverse_typemap[name] +from functools import wraps + +def unpack_args(func): + func_params = inspect.signature(func).parameters + last_param = next(reversed(func_params.values())) + if last_param.kind == last_param.VAR_POSITIONAL: + return func + n = len(func_params) + + @wraps(func) + def wrapper(*args): + if len(args) > n: + warnings.warn("This pickle was created with a different version of dill. " + "Extra arguments to the object constructor were discarded.") + return func(*args[:n]) + return wrapper + +@unpack_args def _create_type(typeobj, *args): return typeobj(*args) -def _create_function(fcode, fglobals, fname=None, fdefaults=None, - fclosure=None, fdict=None, fkwdefaults=None): - # same as FunctionType, but enable passing __dict__ to new function, - # __dict__ is the storehouse for attributes added after function creation - func = FunctionType(fcode, fglobals or dict(), fname, fdefaults, fclosure) - if fdict is not None: - func.__dict__.update(fdict) #XXX: better copy? option to copy? - if fkwdefaults is not None: - func.__kwdefaults__ = fkwdefaults - # 'recurse' only stores referenced modules/objects in fglobals, - # thus we need to make sure that we have __builtins__ as well - if "__builtins__" not in func.__globals__: - func.__globals__["__builtins__"] = globals()["__builtins__"] - # assert id(fglobals) == id(func.__globals__) - return func - +@unpack_args def _create_code(*args): if PY3 and hasattr(args[-3], 'encode'): #FIXME: from PY2 fails (optcode) args = list(args) @@ -815,6 +815,7 @@ def _create_code(*args): elif len(args) == 15: return CodeType(args[0], *args[2:]) return CodeType(*args) +@unpack_args def _create_ftype(ftypeobj, func, args, kwds): if kwds is None: kwds = {} @@ -822,6 +823,7 @@ def _create_ftype(ftypeobj, func, args, kwds): args = () return ftypeobj(func, *args, **kwds) +@unpack_args def _create_lock(locked, *args): #XXX: ignores 'blocking' from threading import Lock lock = Lock() @@ -830,6 +832,7 @@ def _create_lock(locked, *args): #XXX: ignores 'blocking' raise UnpicklingError("Cannot acquire lock") return lock +@unpack_args def _create_rlock(count, owner, *args): #XXX: ignores 'blocking' lock = RLockType() if owner is not None: @@ -839,6 +842,7 @@ def _create_rlock(count, owner, *args): #XXX: ignores 'blocking' return lock # thanks to matsjoyce for adding all the different file modes +@unpack_args def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdata): # buffering=0 # only pickles the handle, not the file contents... good? or StringIO(data)? # (for file contents see: http://effbot.org/librarybook/copy-reg.htm) @@ -931,12 +935,14 @@ class PyObject(ctypes.Structure): f.seek(position) return f +@unpack_args def _create_stringi(value, position, closed): f = StringIO(value) if closed: f.close() else: f.seek(position) return f +@unpack_args def _create_stringo(value, position, closed): f = StringIO() if closed: f.close() @@ -991,18 +997,20 @@ def __ror__(self, a): _CELL_EMPTY = Sentinel('_CELL_EMPTY') if PY3: + @unpack_args def _create_cell(contents=None): if contents is not _CELL_EMPTY: value = contents return (lambda: value).__closure__[0] else: + @unpack_args def _create_cell(contents=None): if contents is not _CELL_EMPTY: value = contents return (lambda: value).func_closure[0] - +@unpack_args def _create_weakref(obj, *args): from weakref import ref if obj is None: # it's dead @@ -1013,6 +1021,7 @@ def _create_weakref(obj, *args): return ref(UserDict(), *args) return ref(obj, *args) +@unpack_args def _create_weakproxy(obj, callable=False, *args): from weakref import proxy if obj is None: # it's dead @@ -1027,6 +1036,7 @@ def _create_weakproxy(obj, callable=False, *args): def _eval_repr(repr_str): return eval(repr_str) +@unpack_args def _create_array(f, args, state, npdict=None): #array = numpy.core.multiarray._reconstruct(*args) array = f(*args) @@ -1035,6 +1045,7 @@ def _create_array(f, args, state, npdict=None): array.__dict__.update(npdict) return array +@unpack_args def _create_dtypemeta(scalar_type): if NumpyDType is True: __hook__() # a bit hacky I think if scalar_type is None: @@ -1042,6 +1053,7 @@ def _create_dtypemeta(scalar_type): return type(NumpyDType(scalar_type)) if OLD37: + @unpack_args def _create_namedtuple(name, fieldnames, modulename, defaults=None): class_ = _import_module(modulename + '.' + name, safe=True) if class_ is not None: @@ -1051,6 +1063,7 @@ def _create_namedtuple(name, fieldnames, modulename, defaults=None): t.__module__ = modulename return t else: + @unpack_args def _create_namedtuple(name, fieldnames, modulename, defaults=None): class_ = _import_module(modulename + '.' + name, safe=True) if class_ is not None: @@ -1163,6 +1176,9 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO else: pickler.write('0') +#def _unmarshal(string): +# return marshal.loads(string) +# #@register(CodeType) #def save_code(pickler, obj): # log.info("Co: %s" % obj) @@ -1905,6 +1921,7 @@ def save_function(pickler, obj): # recurse to get all globals referred to by obj from .detect import globalvars globs_copy = globalvars(obj, recurse=True, builtin=True) + globs_copy.setdefault('__builtins__', __builtin__) # Add the name of the module to the globs dictionary to prevent # the duplication of the dictionary. Pickle the unpopulated @@ -1960,7 +1977,7 @@ def save_function(pickler, obj): if state_dict: state = state, state_dict - _save_with_postproc(pickler, (_create_function, ( + _save_with_postproc(pickler, (FunctionType, ( obj.__code__, globs, obj.__name__, obj.__defaults__, closure ), state), obj=obj, postproc_list=postproc_list) @@ -1973,7 +1990,7 @@ def save_function(pickler, obj): if obj.__dict__: postproc_list.append((setattr, (obj, '__dict__', obj.__dict__))) - _save_with_postproc(pickler, (_create_function, ( + _save_with_postproc(pickler, (FunctionType, ( obj.func_code, globs, obj.func_name, obj.func_defaults, closure )), obj=obj, postproc_list=postproc_list) diff --git a/dill/_shims.py b/dill/_shims.py index 6bda5136..161a5645 100644 --- a/dill/_shims.py +++ b/dill/_shims.py @@ -264,3 +264,22 @@ def _delattr(cell, name): _setattr = Getattr(_dill, '_setattr', setattr) _delattr = Getattr(_dill, '_delattr', delattr) + + +# Kept for backward compatibility, substituted by bare FunctionType. +@move_to(_dill) +def _create_function(fcode, fglobals, fname=None, fdefaults=None, + fclosure=None, fdict=None, fkwdefaults=None): + # 'recurse' only stores referenced modules/objects in fglobals, + # thus we need to make sure that we have __builtins__ as well + if fglobals is None: + fglobals = {} + fglobals.setdefault('__builtins__', globals()['__builtins__']) + # same as FunctionType, but enable passing __dict__ to new function, + # __dict__ is the storehouse for attributes added after function creation + func = _dill.FunctionType(fcode, fglobals, fname, fdefaults, fclosure) + if fdict: + func.__dict__.update(fdict) #XXX: better copy? option to copy? + if fkwdefaults is not None: + func.__kwdefaults__ = fkwdefaults + return func diff --git a/tests/test_functions.py b/tests/test_functions.py index ec9670e2..a78b6613 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -8,6 +8,7 @@ import functools import dill import sys +from dill._dill import _create_function dill.settings['recurse'] = True @@ -102,5 +103,26 @@ def test_functions(): assert dill.loads(dumped_func_e)(1, 2, 3, e2=4) == 12 assert dill.loads(dumped_func_e)(1, 2, 3, e2=4, e3=5) == 15''') +def test_legacy_constructor(): + if is_py3(): + fields = ('__code__', '__globals__', '__name__', '__defaults__', + '__closure__', '__dict__', '__kwdefaults__') + else: + fields = ('func_code', 'func_globals', 'func_name', 'func_defaults', + 'func_closure', '__dict__') + members = [getattr(function_d, f) for f in fields] + + # Old pickle. + old_func = _create_function(*members) + # Recent pickle (dill>=0.3.5). + new_func = _create_function(*members[:5]) + setattr(new_func, fields[5], members[5]) + if is_py3(): + setattr(new_func, fields[6], members[6]) + + for field in fields: + assert getattr(old_func, field) == getattr(new_func, field) + if __name__ == '__main__': test_functions() + test_legacy_constructor()