some new features

.venv/lib/python3.12/site-packages/joblib/externals/loky/backend/reduction.py

@@ -0,0 +1,224 @@
+###############################################################################
+# Customizable Pickler with some basic reducers
+#
+# author: Thomas Moreau
+#
+# adapted from multiprocessing/reduction.py (17/02/2017)
+#  * Replace the ForkingPickler with a similar _LokyPickler,
+#  * Add CustomizableLokyPickler to allow customizing pickling process
+#    on the fly.
+#
+import copyreg
+import io
+import functools
+import types
+import sys
+import os
+
+from multiprocessing import util
+from pickle import loads, HIGHEST_PROTOCOL
+
+###############################################################################
+# Enable custom pickling in Loky.
+
+_dispatch_table = {}
+
+
+def register(type_, reduce_function):
+    _dispatch_table[type_] = reduce_function
+
+
+###############################################################################
+# Registers extra pickling routines to improve picklization  for loky
+
+
+# make methods picklable
+def _reduce_method(m):
+    if m.__self__ is None:
+        return getattr, (m.__class__, m.__func__.__name__)
+    else:
+        return getattr, (m.__self__, m.__func__.__name__)
+
+
+class _C:
+    def f(self):
+        pass
+
+    @classmethod
+    def h(cls):
+        pass
+
+
+register(type(_C().f), _reduce_method)
+register(type(_C.h), _reduce_method)
+
+
+if not hasattr(sys, "pypy_version_info"):
+    # PyPy uses functions instead of method_descriptors and wrapper_descriptors
+    def _reduce_method_descriptor(m):
+        return getattr, (m.__objclass__, m.__name__)
+
+    register(type(list.append), _reduce_method_descriptor)
+    register(type(int.__add__), _reduce_method_descriptor)
+
+
+# Make partial func pickable
+def _reduce_partial(p):
+    return _rebuild_partial, (p.func, p.args, p.keywords or {})
+
+
+def _rebuild_partial(func, args, keywords):
+    return functools.partial(func, *args, **keywords)
+
+
+register(functools.partial, _reduce_partial)
+
+if sys.platform != "win32":
+    from ._posix_reduction import _mk_inheritable  # noqa: F401
+else:
+    from . import _win_reduction  # noqa: F401
+
+# global variable to change the pickler behavior
+try:
+    from joblib.externals import cloudpickle  # noqa: F401
+
+    DEFAULT_ENV = "cloudpickle"
+except ImportError:
+    # If cloudpickle is not present, fallback to pickle
+    DEFAULT_ENV = "pickle"
+
+ENV_LOKY_PICKLER = os.environ.get("LOKY_PICKLER", DEFAULT_ENV)
+_LokyPickler = None
+_loky_pickler_name = None
+
+
+def set_loky_pickler(loky_pickler=None):
+    global _LokyPickler, _loky_pickler_name
+
+    if loky_pickler is None:
+        loky_pickler = ENV_LOKY_PICKLER
+
+    loky_pickler_cls = None
+
+    # The default loky_pickler is cloudpickle
+    if loky_pickler in ["", None]:
+        loky_pickler = "cloudpickle"
+
+    if loky_pickler == _loky_pickler_name:
+        return
+
+    if loky_pickler == "cloudpickle":
+        from joblib.externals.cloudpickle import CloudPickler as loky_pickler_cls
+    else:
+        try:
+            from importlib import import_module
+
+            module_pickle = import_module(loky_pickler)
+            loky_pickler_cls = module_pickle.Pickler
+        except (ImportError, AttributeError) as e:
+            extra_info = (
+                "\nThis error occurred while setting loky_pickler to"
+                f" '{loky_pickler}', as required by the env variable "
+                "LOKY_PICKLER or the function set_loky_pickler."
+            )
+            e.args = (e.args[0] + extra_info,) + e.args[1:]
+            e.msg = e.args[0]
+            raise e
+
+    util.debug(
+        f"Using '{loky_pickler if loky_pickler else 'cloudpickle'}' for "
+        "serialization."
+    )
+
+    class CustomizablePickler(loky_pickler_cls):
+        _loky_pickler_cls = loky_pickler_cls
+
+        def _set_dispatch_table(self, dispatch_table):
+            for ancestor_class in self._loky_pickler_cls.mro():
+                dt_attribute = getattr(ancestor_class, "dispatch_table", None)
+                if isinstance(dt_attribute, types.MemberDescriptorType):
+                    # Ancestor class (typically _pickle.Pickler) has a
+                    # member_descriptor for its "dispatch_table" attribute. Use
+                    # it to set the dispatch_table as a member instead of a
+                    # dynamic attribute in the __dict__ of the instance,
+                    # otherwise it will not be taken into account by the C
+                    # implementation of the dump method if a subclass defines a
+                    # class-level dispatch_table attribute as was done in
+                    # cloudpickle 1.6.0:
+                    # https://github.com/joblib/loky/pull/260
+                    dt_attribute.__set__(self, dispatch_table)
+                    break
+
+            # On top of member descriptor set, also use setattr such that code
+            # that directly access self.dispatch_table gets a consistent view
+            # of the same table.
+            self.dispatch_table = dispatch_table
+
+        def __init__(self, writer, reducers=None, protocol=HIGHEST_PROTOCOL):
+            loky_pickler_cls.__init__(self, writer, protocol=protocol)
+            if reducers is None:
+                reducers = {}
+
+            if hasattr(self, "dispatch_table"):
+                # Force a copy that we will update without mutating the
+                # any class level defined dispatch_table.
+                loky_dt = dict(self.dispatch_table)
+            else:
+                # Use standard reducers as bases
+                loky_dt = copyreg.dispatch_table.copy()
+
+            # Register loky specific reducers
+            loky_dt.update(_dispatch_table)
+
+            # Set the new dispatch table, taking care of the fact that we
+            # need to use the member_descriptor when we inherit from a
+            # subclass of the C implementation of the Pickler base class
+            # with an class level dispatch_table attribute.
+            self._set_dispatch_table(loky_dt)
+
+            # Register the reducers
+            for type, reduce_func in reducers.items():
+                self.register(type, reduce_func)
+
+        def register(self, type, reduce_func):
+            """Attach a reducer function to a given type in the dispatch table."""
+            self.dispatch_table[type] = reduce_func
+
+    _LokyPickler = CustomizablePickler
+    _loky_pickler_name = loky_pickler
+
+
+def get_loky_pickler_name():
+    global _loky_pickler_name
+    return _loky_pickler_name
+
+
+def get_loky_pickler():
+    global _LokyPickler
+    return _LokyPickler
+
+
+# Set it to its default value
+set_loky_pickler()
+
+
+def dump(obj, file, reducers=None, protocol=None):
+    """Replacement for pickle.dump() using _LokyPickler."""
+    global _LokyPickler
+    _LokyPickler(file, reducers=reducers, protocol=protocol).dump(obj)
+
+
+def dumps(obj, reducers=None, protocol=None):
+    global _LokyPickler
+
+    buf = io.BytesIO()
+    dump(obj, buf, reducers=reducers, protocol=protocol)
+    return buf.getbuffer()
+
+
+__all__ = ["dump", "dumps", "loads", "register", "set_loky_pickler"]
+
+if sys.platform == "win32":
+    from multiprocessing.reduction import duplicate
+
+    __all__ += ["duplicate"]