FM / iter

/usr/local/lib64/python3.6/site-packages/torch/utils/data/datapipes/iter/

/usr/local/lib64/python3.6/site-packages/torch/utils/data/datapipes/iter

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
callable.py	8953	0644	edit dl rm
combinatorics.py	4337	0644	edit dl rm
combining.py	16028	0644	edit dl rm
filelister.py	1385	0644	edit dl rm
fileloader.py	1911	0644	edit dl rm
grouping.py	12920	0644	edit dl rm
httpreader.py	1764	0644	edit dl rm
linereader.py	732	0644	edit dl rm
routeddecoder.py	2354	0644	edit dl rm
selecting.py	6018	0644	edit dl rm
streamreader.py	822	0644	edit dl rm
tararchivereader.py	2921	0644	edit dl rm
utils.py	1501	0644	edit dl rm
ziparchivereader.py	2921	0644	edit dl rm
__init__.py	2463	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/utils/data/datapipes/iter/httpreader.py (1764B)

class HTTPReaderIterDataPipe(IterDataPipe[Tuple[str, IOBase]]):
    r""" :class:`HTTPReaderIterDataPipe`

Iterable DataPipe to load file url(s) (http url(s) pointing to file(s)),
    yield file url and IO stream in a tuple

Args:
        datapipe: Iterable DataPipe providing urls
        timeout: Timeout for http request
    """

def __init__(self, datapipe, timeout=None):
        self.datapipe = datapipe
        self.timeout = timeout
        deprecation_warning_torchdata(type(self).__name__)

def __iter__(self):
        for furl in self.datapipe:
            try:
                if self.timeout is None:
                    r = urllib.urlopen(furl)
                else:
                    r = urllib.urlopen(furl, timeout=self.timeout)

yield(furl, r)
            except HTTPError as e:
                raise Exception("Could not get the file.\
                                [HTTP Error] {code}: {reason}."
                                .format(code=e.code, reason=e.reason))
            except URLError as e:
                raise Exception("Could not get the file at {url}.\
                                 [URL Error] {reason}."
                                .format(reason=e.reason, url=furl))
            except Exception:
                raise

def __len__(self) -> int:
        if isinstance(self.datapipe, Sized):
            return len(self.datapipe)
        raise TypeError("{} instance doesn't have valid length".format(type(self).__name__))

from io import IOBase from typing import Sized, Tuple from urllib.error import HTTPError, URLError import urllib.request as urllib from torch.utils.data import IterDataPipe from torch.utils.data.datapipes.utils.common import deprecation_warning_torchdata class HTTPReaderIterDataPipe(IterDataPipe[Tuple[str, IOBase]]): r""" :class:`HTTPReaderIterDataPipe` Iterable DataPipe to load file url(s) (http url(s) pointing to file(s)), yield file url and IO stream in a tuple Args: datapipe: Iterable DataPipe providing urls timeout: Timeout for http request """ def __init__(self, datapipe, timeout=None): self.datapipe = datapipe self.timeout = timeout deprecation_warning_torchdata(type(self).__name__) def __iter__(self): for furl in self.datapipe: try: if self.timeout is None: r = urllib.urlopen(furl) else: r = urllib.urlopen(furl, timeout=self.timeout) yield(furl, r) except HTTPError as e: raise Exception("Could not get the file.\ [HTTP Error] {code}: {reason}." .format(code=e.code, reason=e.reason)) except URLError as e: raise Exception("Could not get the file at {url}.\ [URL Error] {reason}." .format(reason=e.reason, url=furl)) except Exception: raise def __len__(self) -> int: if isinstance(self.datapipe, Sized): return len(self.datapipe) raise TypeError("{} instance doesn't have valid length".format(type(self).__name__))