FM / iter

/usr/local/lib64/python3.6/site-packages/torch/utils/data/datapipes/iter/

/usr/local/lib64/python3.6/site-packages/torch/utils/data/datapipes/iter

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
callable.py	8953	0644	edit dl rm
combinatorics.py	4337	0644	edit dl rm
combining.py	16028	0644	edit dl rm
filelister.py	1385	0644	edit dl rm
fileloader.py	1911	0644	edit dl rm
grouping.py	12920	0644	edit dl rm
httpreader.py	1764	0644	edit dl rm
linereader.py	732	0644	edit dl rm
routeddecoder.py	2354	0644	edit dl rm
selecting.py	6018	0644	edit dl rm
streamreader.py	822	0644	edit dl rm
tararchivereader.py	2921	0644	edit dl rm
utils.py	1501	0644	edit dl rm
ziparchivereader.py	2921	0644	edit dl rm
__init__.py	2463	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/utils/data/datapipes/iter/utils.py (1501B)

import copy
import warnings
from torch.utils.data import IterDataPipe

class IterableWrapperIterDataPipe(IterDataPipe):
    r""":class:`IterableWrapperIterDataPipe`.

Iterable datapipe that wraps an iterable object.

Args:
        iterable: Iterable object to be wrapped into an IterDataPipe
        deepcopy: Option to deepcopy input iterable object for each
            iteration.

.. note::
      If `deepcopy` is set to False explicitly, users should ensure
      that data pipeline doesn't contain any in-place operations over
      the iterable instance, in order to prevent data inconsistency
      across iterations.
    """
    def __init__(self, iterable, deepcopy=True):
        self.iterable = iterable
        self.deepcopy = deepcopy

def __iter__(self):
        source_data = self.iterable
        if self.deepcopy:
            try:
                source_data = copy.deepcopy(self.iterable)
            # For the case that data cannot be deep-copied,
            # all in-place operations will affect iterable variable.
            # When this DataPipe is iterated second time, it will
            # yield modified items.
            except TypeError:
                warnings.warn(
                    "The input iterable can not be deepcopied, "
                    "please be aware of in-place modification would affect source data"
                )
        for data in source_data:
            yield data

def __len__(self):
        return len(self.iterable)

import copy import warnings from torch.utils.data import IterDataPipe class IterableWrapperIterDataPipe(IterDataPipe): r""":class:`IterableWrapperIterDataPipe`. Iterable datapipe that wraps an iterable object. Args: iterable: Iterable object to be wrapped into an IterDataPipe deepcopy: Option to deepcopy input iterable object for each iteration. .. note:: If `deepcopy` is set to False explicitly, users should ensure that data pipeline doesn't contain any in-place operations over the iterable instance, in order to prevent data inconsistency across iterations. """ def __init__(self, iterable, deepcopy=True): self.iterable = iterable self.deepcopy = deepcopy def __iter__(self): source_data = self.iterable if self.deepcopy: try: source_data = copy.deepcopy(self.iterable) # For the case that data cannot be deep-copied, # all in-place operations will affect iterable variable. # When this DataPipe is iterated second time, it will # yield modified items. except TypeError: warnings.warn( "The input iterable can not be deepcopied, " "please be aware of in-place modification would affect source data" ) for data in source_data: yield data def __len__(self): return len(self.iterable)