FM / iter

/usr/local/lib64/python3.6/site-packages/torch/utils/data/datapipes/iter/

/usr/local/lib64/python3.6/site-packages/torch/utils/data/datapipes/iter

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
callable.py	8953	0644	edit dl rm
combinatorics.py	4337	0644	edit dl rm
combining.py	16028	0644	edit dl rm
filelister.py	1385	0644	edit dl rm
fileloader.py	1911	0644	edit dl rm
grouping.py	12920	0644	edit dl rm
httpreader.py	1764	0644	edit dl rm
linereader.py	732	0644	edit dl rm
routeddecoder.py	2354	0644	edit dl rm
selecting.py	6018	0644	edit dl rm
streamreader.py	822	0644	edit dl rm
tararchivereader.py	2921	0644	edit dl rm
utils.py	1501	0644	edit dl rm
ziparchivereader.py	2921	0644	edit dl rm
__init__.py	2463	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/utils/data/datapipes/iter/routeddecoder.py (2354B)

from io import BufferedIOBase
from typing import Any, Callable, Iterable, Iterator, Sized, Tuple

from torch.utils.data import IterDataPipe, functional_datapipe
from torch.utils.data.datapipes.utils.decoder import (
    Decoder,
    basichandlers as decoder_basichandlers,
    imagehandler as decoder_imagehandler,
    extension_extract_fn
)

@functional_datapipe('decode')
class RoutedDecoderIterDataPipe(IterDataPipe[Tuple[str, Any]]):
    r""" :class:`RoutedDecoderIterDataPipe`.

Iterable datapipe to decode binary streams from input DataPipe, yield pathname
    and decoded data in a tuple.

Args:
        datapipe: Iterable datapipe that provides pathname and binary stream in tuples
        handlers: Optional user defined decoder handlers. If None, basic and image decoder
            handlers will be set as default. If multiple handles are provided, the priority
            order follows the order of handlers (the first handler has the top priority)
        key_fn: Function for decoder to extract key from pathname to dispatch handlers.
            Default is set to extract file extension from pathname

Note:
        When `key_fn` is specified returning anything other than extension, the default
        handler will not work and users need to specify custom handler. Custom handler
        could use regex to determine the eligibility to handle data.
    """

def __init__(self,
                 datapipe: Iterable[Tuple[str, BufferedIOBase]],
                 *handlers: Callable,
                 key_fn: Callable = extension_extract_fn) -> None:
        super().__init__()
        self.datapipe: Iterable[Tuple[str, BufferedIOBase]] = datapipe
        if not handlers:
            handlers = (decoder_basichandlers, decoder_imagehandler('torch'))
        self.decoder = Decoder(*handlers, key_fn=key_fn)

def add_handler(self, *handler: Callable) -> None:
        self.decoder.add_handler(*handler)

def __iter__(self) -> Iterator[Tuple[str, Any]]:
        for data in self.datapipe:
            pathname = data[0]
            result = self.decoder(data)
            yield (pathname, result[pathname])

def __len__(self) -> int:
        if isinstance(self.datapipe, Sized):
            return len(self.datapipe)
        raise TypeError("{} instance doesn't have valid length".format(type(self).__name__))

from io import BufferedIOBase from typing import Any, Callable, Iterable, Iterator, Sized, Tuple from torch.utils.data import IterDataPipe, functional_datapipe from torch.utils.data.datapipes.utils.decoder import ( Decoder, basichandlers as decoder_basichandlers, imagehandler as decoder_imagehandler, extension_extract_fn ) @functional_datapipe('decode') class RoutedDecoderIterDataPipe(IterDataPipe[Tuple[str, Any]]): r""" :class:`RoutedDecoderIterDataPipe`. Iterable datapipe to decode binary streams from input DataPipe, yield pathname and decoded data in a tuple. Args: datapipe: Iterable datapipe that provides pathname and binary stream in tuples handlers: Optional user defined decoder handlers. If None, basic and image decoder handlers will be set as default. If multiple handles are provided, the priority order follows the order of handlers (the first handler has the top priority) key_fn: Function for decoder to extract key from pathname to dispatch handlers. Default is set to extract file extension from pathname Note: When `key_fn` is specified returning anything other than extension, the default handler will not work and users need to specify custom handler. Custom handler could use regex to determine the eligibility to handle data. """ def __init__(self, datapipe: Iterable[Tuple[str, BufferedIOBase]], *handlers: Callable, key_fn: Callable = extension_extract_fn) -> None: super().__init__() self.datapipe: Iterable[Tuple[str, BufferedIOBase]] = datapipe if not handlers: handlers = (decoder_basichandlers, decoder_imagehandler('torch')) self.decoder = Decoder(*handlers, key_fn=key_fn) def add_handler(self, *handler: Callable) -> None: self.decoder.add_handler(*handler) def __iter__(self) -> Iterator[Tuple[str, Any]]: for data in self.datapipe: pathname = data[0] result = self.decoder(data) yield (pathname, result[pathname]) def __len__(self) -> int: if isinstance(self.datapipe, Sized): return len(self.datapipe) raise TypeError("{} instance doesn't have valid length".format(type(self).__name__))