FM / multiprocessing

/usr/local/lib64/python3.6/site-packages/torch/multiprocessing/

/usr/local/lib64/python3.6/site-packages/torch/multiprocessing

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
pool.py	1609	0644	edit dl rm
queue.py	1514	0644	edit dl rm
reductions.py	14731	0644	edit dl rm
spawn.py	8399	0644	edit dl rm
_atfork.py	788	0644	edit dl rm
__init__.py	2463	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/multiprocessing/queue.py (1514B)

import io
import multiprocessing.queues
from multiprocessing.reduction import ForkingPickler
import pickle

class ConnectionWrapper(object):
    """Proxy class for _multiprocessing.Connection which uses ForkingPickler to
    serialize objects"""

def __init__(self, conn):
        self.conn = conn

def send(self, obj):
        buf = io.BytesIO()
        ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj)
        self.send_bytes(buf.getvalue())

def recv(self):
        buf = self.recv_bytes()
        return pickle.loads(buf)

def __getattr__(self, name):
        if 'conn' in self.__dict__:
            return getattr(self.conn, name)
        raise AttributeError("'{}' object has no attribute '{}'".format(
            type(self).__name__, 'conn'))

class Queue(multiprocessing.queues.Queue):

def __init__(self, *args, **kwargs):
        super(Queue, self).__init__(*args, **kwargs)
        self._reader: ConnectionWrapper = ConnectionWrapper(self._reader)
        self._writer: ConnectionWrapper = ConnectionWrapper(self._writer)
        self._send = self._writer.send
        self._recv = self._reader.recv

class SimpleQueue(multiprocessing.queues.SimpleQueue):

def _make_methods(self):
        if not isinstance(self._reader, ConnectionWrapper):
            self._reader: ConnectionWrapper = ConnectionWrapper(self._reader)
            self._writer: ConnectionWrapper = ConnectionWrapper(self._writer)
        super(SimpleQueue, self)._make_methods()  # type: ignore[misc]

import io import multiprocessing.queues from multiprocessing.reduction import ForkingPickler import pickle class ConnectionWrapper(object): """Proxy class for _multiprocessing.Connection which uses ForkingPickler to serialize objects""" def __init__(self, conn): self.conn = conn def send(self, obj): buf = io.BytesIO() ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj) self.send_bytes(buf.getvalue()) def recv(self): buf = self.recv_bytes() return pickle.loads(buf) def __getattr__(self, name): if 'conn' in self.__dict__: return getattr(self.conn, name) raise AttributeError("'{}' object has no attribute '{}'".format( type(self).__name__, 'conn')) class Queue(multiprocessing.queues.Queue): def __init__(self, *args, **kwargs): super(Queue, self).__init__(*args, **kwargs) self._reader: ConnectionWrapper = ConnectionWrapper(self._reader) self._writer: ConnectionWrapper = ConnectionWrapper(self._writer) self._send = self._writer.send self._recv = self._reader.recv class SimpleQueue(multiprocessing.queues.SimpleQueue): def _make_methods(self): if not isinstance(self._reader, ConnectionWrapper): self._reader: ConnectionWrapper = ConnectionWrapper(self._reader) self._writer: ConnectionWrapper = ConnectionWrapper(self._writer) super(SimpleQueue, self)._make_methods() # type: ignore[misc]