/usr/local/lib/python3.6/site-packages/fsspec/__pycache__/
/usr/local/lib/python3.6/site-packages/fsspec/__pycache__
NameSizeModeActions
archive.cpython-36.pyc26790644editdlrm
asyn.cpython-36.pyc232500644editdlrm
caching.cpython-36.pyc133250644editdlrm
callbacks.cpython-36.pyc59270644editdlrm
compression.cpython-36.pyc46450644editdlrm
config.cpython-36.pyc27400644editdlrm
conftest.cpython-36.pyc13610644editdlrm
core.cpython-36.pyc202340644editdlrm
dircache.cpython-36.pyc32920644editdlrm
exceptions.cpython-36.pyc7330644editdlrm
fuse.cpython-36.pyc101100644editdlrm
gui.cpython-36.pyc142140644editdlrm
mapping.cpython-36.pyc82080644editdlrm
parquet.cpython-36.pyc111810644editdlrm
registry.cpython-36.pyc76210644editdlrm
spec.cpython-36.pyc522480644editdlrm
transaction.cpython-36.pyc31780644editdlrm
utils.cpython-36.pyc144030644editdlrm
_version.cpython-36.pyc4640644editdlrm
__init__.cpython-36.pyc13940644editdlrm
Edit: /usr/local/lib/python3.6/site-packages/fsspec/__pycache__/spec.cpython-36.pyc (52248B)
3 8%Eg@sddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZmZddlmZdd lmZdd lmZmZmZmZmZejd Zd d ZGdddeZGdddedZ Gdddej!Z"dS)N)ESPIPE) has_magic)sha256)_DEFAULT_CALLBACK) apply_configconf)DirCache) Transaction)_unstrip_protocol other_paths read_blockstringify_pathtokenizeZfsspeccCs |||S)N)clsargskwargsrr5/usr/local/lib/python3.6/site-packages/fsspec/spec.py make_instancesrcs,eZdZdZfddZfddZZS)_Cacheda Metaclass for caching file system instances. Notes ----- Instances are cached according to * The values of the class attributes listed in `_extra_tokenize_attributes` * The arguments passed to ``__init__``. This creates an additional reference to the filesystem, which prevents the filesystem from being garbage collected when all *user* references go away. A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* be made for a filesystem instance to be garbage collected. cs8tj||tjdr$tj|_ni|_tj|_ dS)NZweakref_instance_cache) super__init__rgetweakrefWeakValueDictionary_cacheosgetpid_pid)rrr) __class__rrr)s   z_Cached.__init__cst|}tfddjD}tjtjf|||}|jdd}tj jkrlj j tj _| rj r|j krj |St j||}||_||_||_|jrddlm}||j r| r|j |<|SdS)Nc3s|]}t|dVqdS)N)getattr).0attr)rrr 8sz#_Cached.__call__..Zskip_instance_cacheFr)mirror_sync_methods)rtuple_extra_tokenize_attributesrr threading get_identpoprrrclearcachabler__call__ _fs_token_ storage_argsstorage_options async_implZasynr%)rrrZ extra_tokenstokenskipobjr%)r )rrr-5s*        z_Cached.__call__)__name__ __module__ __qualname____doc__rr- __classcell__rr)r rrs rc@seZdZdZdZdZdZdZdZdZ dZ fZ d d Z e d d Zd dZddZddZddZeddZeddZeddZe ddZddZdd Zdd"d#Zdd$d%Zdd&d'Zd(d)Zdd*d+Zd,d-Z dd.d/Z!dd0d1Z"dd2d3Z#d4d5Z$d6d7Z%d8d9Z&d:d;Z'dd?Z)d@dAZ*dBdCZ+dDdEZ,ddFdGZ-dHdIZ.ddJdKZ/ddLdMZ0ddOdPZ1e2fdQdRZ3de2fdSdTZ4e2fdUdVZ5de2fdWdXZ6ddZd[Z7dd\d]Z8d^d_Z9dd`daZ:ddbdcZ;ddddeZddjdkZ?edldmZ@ddodpZAddqdrZBddsdtZCdudvZDddwdxZEdydzZFed{d|ZGd}d~ZHdddZIeddZJddZKddZLdddZMdddZNdddZOddZPddZQddZRdddZSddZTdddZUdddZVdddZWdddZXddZYd!S)AbstractFileSystemz An abstract super-class for pythonic file-systems Implementations are expected to be compatible with or, better, subclass from here. TF/ZabstractcOsh|jr dSd|_d|_d|_g|_tf||_|jddrFtjdt |jddr^tjdt d|_ dS)aCreate and configure file-system instance Instances may be cachable, so if similar enough arguments are seen a new instance is not required. The token attribute exists to allow implementations to cache instances if they wish. A reasonable default should be provided if there are no arguments. Subclasses should call this method. Parameters ---------- use_listings_cache, listings_expiry_time, max_paths: passed to ``DirCache``, if the implementation supports directory listing caching. Pass use_listings_cache=False to disable such caching. skip_instance_cache: bool If this is a cachable implementation, pass True here to force creating a new instance even if a matching instance exists, and prevent storing this instance. asynchronous: bool loop: asyncio-compatible IOLoop or None NTFZadd_docsz add_docs is no longer supported.Z add_aliaseszadd_aliases has been removed.) _cached_intrans _transaction"_invalidated_caches_in_transactionr dircacher*warningswarn FutureWarningr.)selfrr0rrrrfs     zAbstractFileSystem.__init__cCs|jS)N)r.)rGrrr _fs_tokenszAbstractFileSystem._fs_tokencCs|jS)N)rH)rGrrr__dask_tokenize__sz$AbstractFileSystem.__dask_tokenize__cCs t|jdS)N)intrH)rGrrr__hash__szAbstractFileSystem.__hash__cCst|t|o|j|jkS)N) isinstancetyperH)rGotherrrr__eq__szAbstractFileSystem.__eq__cCstt||j|jffS)N)rrNr/r0)rGrrr __reduce__szAbstractFileSystem.__reduce__cst|trfdd|DSt|}tjtr8jfnj}xR|D]J}|j|drl|t|dd}qD|j|drD|t|dd}qDW|jd}|pjS) zTurn path from fully-qualified to file-system-specific May require FS-specific handling, e.g., for relative paths or links. csg|]}j|qSr)_strip_protocol)r"p)rrr sz6AbstractFileSystem._strip_protocol..z://Nz::r;r=) rMlistrprotocolstr startswithlenrstrip root_marker)rpathprotosrWr)rrrRs   z"AbstractFileSystem._strip_protocolcCsiS)afIf kwargs can be encoded in the paths, extract them here This should happen before instantiation of the class; incoming paths then should be amended to strip the options in methods. Examples may look like an sftp path "sftp://user@host:/my/path", where the user and host should become kwargs and later get stripped. r)r]rrr_get_kwargs_from_urlss z(AbstractFileSystem._get_kwargs_from_urlscCs&t|js|St|jjdSdS)z|Return the most recently created FileSystem If no instance has been created, then create one with defaults rN)rZrrVvalues)rrrrcurrents zAbstractFileSystem.currentcCs|jdkrt||_|jS)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. N)rAr )rGrrr transactions  zAbstractFileSystem.transactioncCsd|_t||_|jS)z@Begin write transaction for deferring files, non-context versionT)r@r rArc)rGrrrstart_transactions z$AbstractFileSystem.start_transactioncCs8|jjd|_x|jD]}|j|qW|jjdS)z-Finish write transaction, non-context versionN)rcZcompleterArBinvalidate_cacher+)rGr]rrrend_transactions   z"AbstractFileSystem.end_transactionNcCs|jr|jj|dS)z Discard any cached directory information Parameters ---------- path: string or None If None, clear all listings cached else listings at or under given path. N)r@rBappend)rGr]rrrresz#AbstractFileSystem.invalidate_cachecKsdS)a Create directory entry at path For systems that don't have true directories, may create an for this instance only and not touch the real filesystem Parameters ---------- path: str location create_parents: bool if True, this is equivalent to ``makedirs`` kwargs: may be permissions, etc. Nr)rGr]create_parentsrrrrmkdirszAbstractFileSystem.mkdircCsdS)aRecursively make directories Creates directory at path and any intervening required directories. Raises exception if, for instance, the path already exists but is a file. Parameters ---------- path: str leaf directory name exist_ok: bool (False) If False, will error if the target already exists Nr)rGr]exist_okrrrmakedirsszAbstractFileSystem.makedirscCsdS)zRemove a directory, if emptyNr)rGr]rrrrmdirszAbstractFileSystem.rmdircKstdS)anList objects at path. This should include subdirectories and files at that location. The difference between a file and a directory must be clear when details are requested. The specific keys, or perhaps a FileInfo class, or similar, is TBD, but must be consistent across implementations. Must include: - full path to the entry (without protocol) - size of the entry, in bytes. If the value cannot be determined, will be ``None``. - type of entry, "file", "directory" or other Additional information may be present, aproriate to the file-system, e.g., generation, checksum, etc. May use refresh=True|False to allow use of self._ls_from_cache to check for a saved listing and avoid calling the backend. This would be common where listing may be expensive. Parameters ---------- path: str detail: bool if True, gives a list of dictionaries, where each is the same as the result of ``info(path)``. If False, gives a list of paths (str). kwargs: may have additional backend-specific options, such as version information Returns ------- List of strings if detail is False, or list of directory information dicts if detail is True. N)NotImplementedError)rGr]detailrrrrlss'zAbstractFileSystem.lsc st|j}jd|jkr*|jjdSy0fdd|j|D}t|dkrXt|Stk rnYnXdS)zCheck cache for listing Returns listing, if found (may me empty list for a directly that exists but contains nothing), None if not in cache. r=cs:g|]2}|dks2|djdkr|ddkr|qS)namer=rN directory)r[)r"f)r]rrrTOs z5AbstractFileSystem._ls_from_cache..rN)_parentr[rCrZFileNotFoundErrorKeyError)rGr]parentfilesr)r]r_ls_from_cacheDs   z!AbstractFileSystem._ls_from_cachec ksN|j|}i}i}i}|jdd}y|j|fddi|}Wn,ttfk rh|r^|iifS|ggfSXxj|D]b} | djd} | jddd } | ddkr| |kr| || <| || <qp| |kr| |d <qp| || <qpW|r|||fVn|t|t|fV|d k r|d8}|dkrd Sx,|D]$} |j| f||d |Ed Hq"Wd S) a?Return all files belows path List all files, recursing into subdirectories; output is iterator-style, like ``os.walk()``. For a simple list of files, ``find()`` is available. Note that the "files" outputted will include anything that is not a directory, such as links. Parameters ---------- path: str Root to recurse into maxdepth: int Maximum recursion depth. None means limitless, but not recommended on link-based file-systems. kwargs: passed to ``ls`` rnFTrpr=rrNrqr>N)maxdepthrnr`) rRr*rortIOErrorr[rsplitrVwalk) rGr]ryrZ full_dirsdirsrwrnlistinginfopathnamerpdrrrr|[s:           zAbstractFileSystem.walkc  s|j|}t|jdd}xL|j||fddi|D]0\}}}|rN|j|jdd|jDq6W r|j|ri|<t} |s| Sfdd| DSdS)aList all files below path. Like posix ``find`` command without conditions Parameters ---------- path : str maxdepth: int or None If not None, the maximum number of levels to descend withdirs: bool Whether to include directory paths in the output. This is True when used by glob, but users usually only want files. kwargs are passed to ``ls``. rnFTcSsi|]\}}||dqS)rpr)r"rprrrr sz+AbstractFileSystem.find..csi|]}||qSrr)r"rp)outrrrsN)rRdictr*r|updateitemsisfilesorted) rGr]rywithdirsrrn_r}rwnamesr)rrfinds  $ zAbstractFileSystem.findcKsVi}x8|j|fd|i|D]}|j|}|d||d<qW|rNt|jS|SdS)aSpace used by files within a path Parameters ---------- path: str total: bool whether to sum all the file sizes maxdepth: int or None maximum number of directory levels to descend, None for unlimited. kwargs: passed to ``ls`` Returns ------- Dict of {fn: size} if total=False, or int otherwise, where numbers refer to bytes used. rysizerpN)rrsumra)rGr]totalryrsizesrrrrrrdus  zAbstractFileSystem.duc sVddl}|jd}|j|}|jddkr4|jdnt|}|jddkrT|jdnt|}|jddkrt|jdnt|}t|||}|jdd} t|s|} d } |r|d 7}n0|j|r| s|gS||j |iSn | sgSiSnd|d|krB|d|j d} |d| d } d |kr&dn|| d dj dd } n,d } d |krTdn||d dj dd } |j| f| d d d|d|j ddj ddj ddj ddj ddj ddj ddj ddj ddj d d!j d"d#j dj ddd|jd$d%|jd&d'|jj d%d(fd)d*tD} | rJ| St| SdS)+a Find files by glob-matching. If the path ends with '/' and does not contain "*", it is essentially the same as ``ls(path)``, returning only files. We support ``"**"``, ``"?"`` and ``"[..]"``. We do not support ^ for pattern negation. Search path names that contain embedded characters special to this implementation of glob may not produce expected results; e.g., 'foo/bar/*starredfilename*'. kwargs are passed to ``ls``. rNr=*?[rnFrz/*z**r>T)ryrrn^\z\\.z\.+z\+z//(z\()z\)|z\|z\^$z\${z\{}z\}z[*]{2}z =PLACEHOLDER=z[*]z[^/]*z.*cs.i|]&}j|jddjdr||qS)z//r=)matchreplacer[)r"rS)allpathspatternrrrsz+AbstractFileSystem.glob..)reendswithrRrrZminr*rexistsrrindexcountrr[subcompilerrV)rGr]rrendsZindstarZindquesZindbraceindrnrootdepthZind2rr)rrrglobsH         *(r  zAbstractFileSystem.globc Ks"y|j|f|dSdSdS)z!Is there a file at the given pathTFN)r)rGr]rrrrr)s zAbstractFileSystem.existscKs |j|S)zEIf there is a file at the given path (including broken links))r)rGr]rrrrlexists2szAbstractFileSystem.lexistsc s|j|j|jfddi|}fdd|D}|rD|dS|jfddi|}jdfdd|D}t|dkrd |dkrd |dd <|dSt|dks|rdd d Std S) agGive details of entry at path Returns a single dictionary, with exactly the same information as ``ls`` would with ``detail=True``. The default implementation should calls ls and could be overridden by a shortcut. kwargs are passed on to ```ls()``. Some file systems might not be able to measure the file's size, in which case, the returned dict will include ``'size': None``. Returns ------- dict with keys: name (full path in the FS), size (in bytes), type (file, directory, or something else) and other FS-specific keys. rnTcs"g|]}|djdkr|qS)rpr=)r[)r"o)r]rrrTJsz+AbstractFileSystem.info..rr=cs"g|]}|djdkr|qS)rpr=)r[)r"r)r]rrrTOsrrNrq)rprrN)rRrorsr[rZrt)rGr]rrZout1r)r]rr7s      zAbstractFileSystem.infocCstt|j|dS)aUnique value for current version of file If the checksum is the same from one moment to another, the contents are guaranteed to be the same. If the checksum changes, the contents *might* have changed. This should normally be overridden; default will probably capture creation/modification timestamp (which would be good) or maybe access timestamp (which would be bad) rJ)rKrr)rGr]rrrchecksumYs zAbstractFileSystem.checksumcCs|j|jddS)zSize in bytes of filerN)rr)rGr]rrrrfszAbstractFileSystem.sizecsfdd|DS)z-Size in bytes of each file in a list of pathscsg|]}j|qSr)r)r"rS)rGrrrTlsz,AbstractFileSystem.sizes..r)rGpathsr)rGrrjszAbstractFileSystem.sizesc Cs,y|j|ddkStk r&dSXdS)zIs this entry directory-like?rNrqFN)rrz)rGr]rrrisdirnszAbstractFileSystem.isdirc Cs"y|j|ddkSdSdS)zIs this entry file-like?rNfileFN)r)rGr]rrrruszAbstractFileSystem.isfilec Ks|j|df|h}|dk rD|dkr.|j|n|jtd|j||dk rp|dkr^|j|}|j||jS|jSQRXdS)ahGet the content of a file Parameters ---------- path: URL of file on this filesystems start, end: int Bytes limits of the read. If negative, backwards from end, like usual python slices. Either can be None for start or end of file, respectively kwargs: passed to ``open()``. rbNr)openseekmaxrreadtell)rGr]startendrrrrrrcat_file|s   zAbstractFileSystem.cat_filec Ks&|j|d}|j|WdQRXdS)zSet the bytes of given filewbN)rwrite)rGr]valuerrrrrr pipe_fileszAbstractFileSystem.pipe_filecKsft|tr"|j|j||f|n@t|trZx4|jD]\}}|j|j||f|q6WntddS)aPut value into path (counterpart to ``cat``) Parameters ---------- path: string or dict(str, bytes) If a string, a single remote location to put ``value`` bytes; if a dict, a mapping of {path: bytesvalue}. value: bytes, optional If using a single path, these are the bytes to put there. Ignored if ``path`` is a dict zpath must be str or dictN)rMrXrrRrr ValueError)rGr]rrkvrrrpipes   zAbstractFileSystem.pipec s|dk r tt|tstt|ts2|gt|}t|tsJ|gt|}t|t|ksjt|t|krntfddt|||DS)Ncs g|]\}}}j|||qSr)r)r"rSse)rGrrrTsz1AbstractFileSystem.cat_ranges..)rmrMrV TypeErrorrZrzip)rGrZstartsrZmax_gaprr)rGr cat_rangess    zAbstractFileSystem.cat_rangesraisecKs|j||d}t|dks6t|ts6|d|j|kri}x`|D]X}y|j|f|||<Wq@tk r}z |dkrv|dkr|||<WYdd}~Xq@Xq@W|S|j|df|SdS)aFetch (potentially multiple) paths' contents Parameters ---------- recursive: bool If True, assume the path(s) are directories, and get all the contained files on_error : "raise", "omit", "return" If raise, an underlying exception will be raised (converted to KeyError if the type is in self.missing_exceptions); if omit, keys with exception will simply not be included in the output; if "return", all keys are included in the output, but the value will be bytes or an exception instance. kwargs: passed to cat_file Returns ------- dict of {path: contents} if there are multiple paths or the path has been otherwise expanded ) recursiverrrreturnN) expand_pathrZrMrVrRr Exception)rGr]ron_errorrrrrrrrcats   zAbstractFileSystem.catc Ks|j|rtj|dddS|j|df|\}|jt|ddt|d4}d}x(|rx|j|j}|j|}|j |qRWWdQRXWdQRXdS)z Copy single remote file to localT)rjNrrr) rrrkrset_sizer!r blocksizerrelative_update) rGrpathlpathcallbackrf1f2data segment_lenrrrget_files    zAbstractFileSystem.get_filec Ksddlm}t|tr||}|j||d}t||}|jt|x:|jt ||D]&\}}|j ||||j ||f|qVWdS)aTCopy file(s) to local. Copies a specific file or tree of files (if recursive=True). If lpath ends with a "/", it will be assumed to be a directory, and target files will go within. Can submit a list of paths, which may be glob-patterns and will be expanded. Calls get_file for each source. r)make_path_posix)rN) implementations.localrrMrXrr rrZwraprbranchr) rGrrrrrrrpathslpathsrrrrs   zAbstractFileSystem.getc Kstjj|r|j|dddSt|d}|jdd}|j||jd|j|jtj |dd|j|df|8}x0|j |kr|j |j }|j |} |j| qxWWdQRXWdQRXdS)zCopy single file to remoteT)rjNrrr;r)rr]rrkrrrmkdirsrsfspathrrrrr) rGrrrrrrrrrrrrput_file s       zAbstractFileSystem.put_filec  sddlm}m}t|tr$j|nfdd|D}t|trH||}|}|j||d} t| |t|totj|d} |j t | x:|j t | | D]&\}}|j |||j||f|qWdS)aCopy file(s) from local. Copies a specific file or tree of files (if recursive=True). If rpath ends with a "/", it will be assumed to be a directory, and target files will go within. Calls put_file for each source. r)LocalFileSystemrcsg|]}j|qSr)rR)r"rS)rGrrrT*sz*AbstractFileSystem.put..)r)rN)rrrrMrXrRrr rrrZrrrr) rGrrrrrrrfsrrr)rGrputs  zAbstractFileSystem.putc Cs"|j|d }|j|SQRXdS)z&Get the first ``size`` bytes from filerN)rr)rGr]rrrrrrhead9szAbstractFileSystem.headc Cs8|j|d"}|jt| |j d|jSQRXdS)z%Get the last ``size`` bytes from filerr;N)rrrrr)rGr]rrrrrrtail>szAbstractFileSystem.tailcKstdS)N)rm)rGpath1path2rrrrcp_fileDszAbstractFileSystem.cp_filec Ks|dkr|rd}n |dkrd}|j||d}t||}xJt||D]<\}}y|j||f|WqBtk r||dkrxYqBXqBWdS)a?Copy within two locations in the filesystem on_error : "raise", "ignore" If raise, any not-found exceptions will be raised; if ignore any not-found exceptions will cause the path to be skipped; defaults to raise unless recursive is true, where the default is ignore Nignorer)r)rr rrrt) rGrrrrrrp1p2rrrcopyGs  zAbstractFileSystem.copycst|trj|g||}n|s$|n|d}t}fdd|D}x|D]}t|rtj|}||O}|rJ|tjt|||dO}qJn |rtj||dd}||O}||krJ|dksʈj|rJ|j |qJW|st |tt |S)zhTurn one or more globs or directories into a list of all matching paths to files or directories.rcsg|]}j|qSr)rR)r"rS)rGrrrTfsz2AbstractFileSystem.expand_path..)rryT)ryrF) rMrXrsetrrrVrraddrtr)rGr]rryrrSbitZrecr)rGrr]s,  zAbstractFileSystem.expand_pathcKs$|j||||d|j||ddS)z)Move file(s) from one location to another)rry)rN)rrm)rGrrrryrrrrmv|szAbstractFileSystem.mvcCs|j|dS)z Delete a fileN)_rm)rGr]rrrrm_fileszAbstractFileSystem.rm_filecCstdS)zDelete one fileN)rm)rGr]rrrrszAbstractFileSystem._rmcCs0|j|||d}xt|D]}|j|qWdS)aDelete files. Parameters ---------- path: str or list of str File(s) to delete. recursive: bool If file(s) are directories, recursively delete contents and then also remove the directory maxdepth: int or None Depth to pass to walk for finding files to delete, if recursive. If None, there will be no limit and infinite recursion may be possible. )rryN)rreversedr)rGr]rryrSrrrrszAbstractFileSystem.rmcCsD|j|jd}d|kr:|jdddj|j}|j|S|jSdS)Nr=rr)rRr[r{lstripr\)rr]rvrrrrss  zAbstractFileSystem._parentrcKst|||||fd|i|S)z4Return raw bytes-mode file-like from the file-system cache_options)AbstractBufferedFile)rGr]mode block_size autocommitrrrrr_opens zAbstractFileSystem._openc sddl}|j|}d|krb|jddd}fdddD}|j|j||f|||d f|Sjd |j } |j|f||| |d } |dk rddlm } ddl m } | ||}| |} | | |dd} | rd|kr|j j j| | SdS)a6 Return a file-like object from the filesystem The resultant instance must function correctly in a context ``with`` block. Parameters ---------- path: str Target file mode: str like 'rb', 'w' See builtin ``open()`` block_size: int Some indication of buffering - this is a value in bytes cache_options : dict, optional Extra arguments to pass through to the cache. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding, errors, newline: passed on to TextIOWrapper for text mode rNbtr>cs i|]}|krj||qSr)r*)r"r)rrrrsz+AbstractFileSystem.open..encodingerrorsnewline)rr compressionr)rrrr)compr)get_compression)rr)rrr)iorRr TextIOWrapperrr*r@rZfsspec.compressionr Z fsspec.corer rcrwrg)rGr]rrrrrr Z text_kwargsacrrr r compressr)rrrs@     zAbstractFileSystem.openc Ks6|s|j| r.|j|df|WdQRXntdS)a!Create empty file, or update timestamp Parameters ---------- path: str file location truncate: bool If True, always set file size to 0; if False, update timestamp and leave file unchanged, if backend allows this rN)rrrm)rGr]truncaterrrrtouchs  zAbstractFileSystem.touchcCstt|j|jjS)z2Hash of file properties, to tell if it has changed)rrXrencode hexdigest)rGr]rrrukeyszAbstractFileSystem.ukeyc CsT|j|d>}|j}|dkr |}|dk r<|||kr<||}t||||SQRXdS)aRead a block of bytes from Starting at ``offset`` of the file, read ``length`` bytes. If ``delimiter`` is set then we ensure that the read starts and stops at delimiter boundaries that follow the locations ``offset`` and ``offset + length``. If ``offset`` is zero then we start at zero. The bytestring returned WILL include the end delimiter string. If offset+length is beyond the eof, reads to eof. Parameters ---------- fn: string Path to filename offset: int Byte offset to start read length: int Number of bytes to read delimiter: bytes (optional) Ensure reading starts and stops at delimiter bytestring Examples -------- >>> fs.read_block('data/file.csv', 0, 13) # doctest: +SKIP b'Alice, 100\nBo' >>> fs.read_block('data/file.csv', 0, 13, delimiter=b'\n') # doctest: +SKIP b'Alice, 100\nBob, 200\n' Use ``length=None`` to read to the end of the file. >>> fs.read_block('data/file.csv', 0, None, delimiter=b'\n') # doctest: +SKIP b'Alice, 100\nBob, 200\nCharlie, 300' See Also -------- utils.read_block rN)rrr )rGfnoffsetlength delimiterrrrrrrr s%zAbstractFileSystem.read_blockcCsbddl}t|}dj|j|jf}t|jttfr<|jdn|j}|j t f|||j d|j S)ar JSON representation of this filesystem instance Returns ------- str: JSON structure with keys cls (the python location of this class), protocol (text name of this class's protocol, first one in case of multiple), args (positional args, usually empty), and all other kwargs as their own keys. rNr)rrWr) jsonrNjoinr6r5rMrWr&rVdumpsrr/r0)rGrrprotorrrto_jsonGs  zAbstractFileSystem.to_jsoncCsvddl}ddlm}m}|j|}|jd}y||jd}Wn$tttt fk rb||}YnX||jdf|S)a/ Recreate a filesystem instance from JSON representation See ``.to_json()`` for the expected structure of the input Parameters ---------- blob: str Returns ------- file system instance, not necessarily of this particular class. rNr) _import_classget_filesystem_classrWrr) rregistryrrloadsr* ImportErrorr RuntimeErrorru)ZblobrrrZdicrWrrrr from_jsonbs  zAbstractFileSystem.from_jsoncCs|S)zW Make a version of the FS instance which will be acceptable to pyarrow r)rGrrr_get_pyarrow_filesystem}sz*AbstractFileSystem._get_pyarrow_filesystemcCsddlm}|||||S)zCreate key/value store based on this file-system Makes a MutableMapping interface to the FS at the given root path. See ``fsspec.mapping.FSMap`` for further details. r)FSMap)mappingr&)rGrcheckcreater&rrr get_mappers zAbstractFileSystem.get_mappercCs|jjdS)a Clear the cache of filesystem instances. Notes ----- Unless overridden by setting the ``cachable`` class attribute to False, the filesystem class stores a reference to newly created instances. This prevents Python's normal rules around garbage collection from working, since the instances refcount will not drop to zero until ``clear_instance_cache`` is called. N)rr+)rrrrclear_instance_caches z'AbstractFileSystem.clear_instance_cachecCstdS)z=Return the created timestamp of a file as a datetime.datetimeN)rm)rGr]rrrcreatedszAbstractFileSystem.createdcCstdS)z>Return the modified timestamp of a file as a datetime.datetimeN)rm)rGr]rrrmodifiedszAbstractFileSystem.modifiedcKs|j|fd|i|S)z$Alias of `AbstractFileSystem.mkdir`.rh)ri)rGr]rhrrrrmakedirszAbstractFileSystem.makedircCs|j||dS)z'Alias of `AbstractFileSystem.makedirs`.)rj)rk)rGr]rjrrrrszAbstractFileSystem.mkdirscKs|j|fd|i|S)z!Alias of `AbstractFileSystem.ls`.rn)ro)rGr]rnrrrrlistdirszAbstractFileSystem.listdircKs|j||f|S)z#Alias of `AbstractFileSystem.copy`.)r)rGrrrrrrcpszAbstractFileSystem.cpcKs|j||f|S)z!Alias of `AbstractFileSystem.mv`.)r)rGrrrrrrmoveszAbstractFileSystem.movecKs|j|f|S)z#Alias of `AbstractFileSystem.info`.)r)rGr]rrrrstatszAbstractFileSystem.statcKs|j|f||d|S)z!Alias of `AbstractFileSystem.du`.)rry)r)rGr]rryrrrr disk_usageszAbstractFileSystem.disk_usagecKs|j||f|S)z!Alias of `AbstractFileSystem.mv`.)r)rGrrrrrrrenameszAbstractFileSystem.renamecCs|j|||dS)z!Alias of `AbstractFileSystem.rm`.)rry)r)rGr]rryrrrdeleteszAbstractFileSystem.deletecKs|j||fd|i|S)z"Alias of `AbstractFileSystem.put`.r)r)rGrrrrrrruploadszAbstractFileSystem.uploadcKs|j||fd|i|S)z"Alias of `AbstractFileSystem.get`.r)r)rGrrrrrrrdownloadszAbstractFileSystem.downloaddcKs tddS)aCreate a signed URL representing the given path Some implementations allow temporary URLs to be generated, as a way of delegating credentials. Parameters ---------- path : str The path on the filesystem expiration : int Number of seconds to enable the URL for (if supported) Returns ------- URL : str The signed URL Raises ------ NotImplementedError : if method is not implemented for a filesystem z+Sign is not implemented for this filesystemN)rm)rGr]Z expirationrrrrsignszAbstractFileSystem.signcCsdS)NFr)rGrrr _isfilestoreszAbstractFileSystem._isfilestorei@)N)T)F)T)N)NF)TN)NN)N)N)Fr)r)r)FN)FN)FN)FN)rNTN)rNNN)T)N)FF)T)F)T)TN)FN)F)F)r8)Zr5r6r7r8r,r?rseprWr1r\r'rpropertyrHrIrLrPrQ classmethodrR staticmethodr_rbrcrdrfrerirkrlrorxr|rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrsrrrrr rr$r%r*r+r,r-r.rr/r0r1r2r3r4r5r6r7r9r:rrrrr:Ss)       ) ; ! X "    (           E  -          r:) metaclassc@sPeZdZdZdQZdZdRd d Zed d Zej dd ZeddZ eddZ e j ddZ ddZ ddZ ddZddZddZddZdSd!d"Zd#d$ZdTd&d'ZdUd(d)Zd*d+Zd,d-ZdWd/d0Zd1d2ZdXd4d5Zd6d7Zd8d9Zd:d;Zdd?Zd@dAZ dBdCZ!dDdEZ"dFdGZ#dHdIZ$dJdKZ%e%Z&dLdMZ'dNdOZ(dS)Yra?Convenient class to derive from to provide buffering In the case that the backend does not provide a pythonic file-like object already, this class contains much of the logic to build one. The only methods that need to be overridden are ``_upload_chunk``, ``_initiate_upload`` and ``_fetch_range``. r;NrdefaultT readaheadc Ksddlm} ||_||_||_|dkr,|jn||_d|_||_d|_ d|_ d|_ |dkr\i}d| kr~t j dt| jd|d<| |_|dkrtd |d kr|dk r||_n |jd |_| ||j|j|jf||_ntj|_d|_d|_d|_dS)a Template for files with buffered reading and writing Parameters ---------- fs: instance of FileSystem path: str location in file-system mode: str Normal file modes. Currently only 'wb', 'ab' or 'rb'. Some file systems may be read-only, and some may not support append. block_size: int Buffer size for reading or writing, 'default' for class default autocommit: bool Whether to write to final destination; may only impact what happens when file is being closed. cache_type: {"readahead", "none", "mmap", "bytes"}, default "readahead" Caching policy in read mode. See the definitions in ``core``. cache_options : dict Additional options passed to the constructor for the cache specified by `cache_type`. size: int If given and in read mode, suppressed having to look up the file size kwargs: Gets stored as self.kwargs r)cachesrBNrFZtrimzyPassing 'trim' to control the cache behavior has been deprecated. Specify it within the 'cache_options' argument instead.abrrzFile mode not supportedr)rBN>rErr)corerDr]rrDEFAULT_BLOCK_SIZErlocrrrclosedrDrErFr*rrmrdetails _fetch_rangecacher BytesIObufferrforcedlocation) rGrr]rrrZ cache_typerrrrDrrrrs<&   zAbstractBufferedFile.__init__cCs |jdkr|jj|j|_|jS)N)_detailsrrr])rGrrrrJQs zAbstractBufferedFile.detailscCs||_|d|_dS)Nr)rQr)rGrrrrrJWscCst|j|jS)N)r r]r)rGrrr full_name\szAbstractBufferedFile.full_namecCs t|ddS)N_closedT)r!)rGrrrrI`szAbstractBufferedFile.closedcCs ||_dS)N)rS)rGcrrrrIfscCs&d|jkrt|Stt|jdSdS)NwrJ)ridrKrrJ)rGrrrrLjs zAbstractBufferedFile.__hash__cCs$|jdko"|jdko"t|t|kS)zAFiles are equal if they have the same checksum, only in read moder)rhash)rGrOrrrrPpszAbstractBufferedFile.__eq__cCsdS)z#Move from temp to final destinationNr)rGrrrcommittszAbstractBufferedFile.commitcCsdS)zThrow away temporary fileNr)rGrrrdiscardwszAbstractBufferedFile.discardcCsd|jkr|jStddS)z File information about this pathr z Info not available while writingN)rrJr)rGrrrrzs zAbstractBufferedFile.infocCs|jS)zCurrent file location)rH)rGrrrrszAbstractBufferedFile.tellrcCszt|}|jdksttd|dkr*|}n4|dkr>|j|}n |dkrR|j|}n td||dkrntd||_|jS)zSet current file location Parameters ---------- loc: int byte location whence: {0, 1, 2} from start of file, current location or end of file, resp. rz Seek only available in read moderrr;z(invalid whence (%s, should be 0, 1 or 2)zSeek before start of file)rKrOSErrorrrHrr)rGrHwhencenlocrrrrs      zAbstractBufferedFile.seekcCsd|jdkrtd|jr td|jr.td|jj|}|j|7_|jj|jkr`|j |S)z Write data to buffer. Buffer only sent on flush() or if buffer is greater than or equal to blocksize. Parameters ---------- data: bytes Set of bytes to be written. rrEzFile not in write modezI/O operation on closed file.z0This file has been force-flushed, can only close>rEr) rrrIrOrNrrHrrflush)rGrrrrrrs  zAbstractBufferedFile.writeFc Cs|jrtd|r |jr td|r*d|_|jd kr8dS| rR|jj|jkrRdS|jdkrd|_y |jWnd|_YnX|j |dd k r|j|jj dd 7_t j |_dS) a{ Write buffered data to backend store. Writes the current buffer, if it is larger than the block-size, or if the file is being closed. Parameters ---------- force: bool When closing, write the last block even if it is smaller than blocks are allowed to be. Disallows further writing to this file. zFlush on closed filez+Force flush cannot be called more than onceTrrENr)finalFr;>rEr) rIrrOrrNrrr_initiate_upload _upload_chunkrr rM)rGforcerrrr]s(    zAbstractBufferedFile.flushcCsdS)zWrite one part of a multi-block file upload Parameters ========== final: bool This is the last block, so should complete file, if self.autocommit is True. Nr)rGr^rrrr`sz"AbstractBufferedFile._upload_chunkcCsdS)zCreate remote file/uploadNr)rGrrrr_sz%AbstractBufferedFile._initiate_uploadcCstdS)z*Get the specified set of bytes from remoteN)rm)rGrrrrrrKsz!AbstractBufferedFile._fetch_rangercCs|dkr d nt|}|jdkr&td|dkr:|j|j}|jrHtdtjd||j|j|f|dkrpdS|jj |j|j|}|jt |7_|S) z Return data from cache, or fetch pieces as necessary Parameters ---------- length: int (-1) Number of bytes to read; if <0, all remaining bytes. NrrzFile not in read moderzI/O operation on closed file.z%s read: %i - %ir`) rKrrrrHrIloggerdebugrLZ_fetchrZ)rGrrrrrrs   zAbstractBufferedFile.readcCs2t|jd}|j|j}||dt|<t|S)zxmirrors builtin file's readinto method https://docs.python.org/3/library/io.html#io.RawIOBase.readinto BN) memoryviewcastrnbytesrZ)rGrrrrrrreadinto s zAbstractBufferedFile.readinto cCsg}xx|j}|j|p|j}t|dkr,P|j|}|dkrp|j|d|t||j||t|P|j|qWdj|S)aReturn data between current position and first occurrence of char char is included in the output, except if the end of the tile is encountered first. Parameters ---------- char: bytes Thing to find blocks: None or int How much to read in each go. Defaults to file blocksize - which may mean a new read on every call. rrNrbr`)rrrrZrrgrr)rGcharblocksrrpartfoundrrr readuntils  zAbstractBufferedFile.readuntilcCs |jdS)zRead until first occurrence of newline character Note that, because of character encoding, this is not necessarily a true line ending. rj)ro)rGrrrreadline2szAbstractBufferedFile.readlinecCs|j}|r|StdS)N)rp StopIteration)rGrrrr__next__:szAbstractBufferedFile.__next__cCs|S)Nr)rGrrr__iter__@szAbstractBufferedFile.__iter__cCsH|j}|jd}dd|ddD}|jdr6|S||dgSdS)z/Return all data, split by the newline characterrjcSsg|] }|dqS)rjr)r"lrrrrTGsz2AbstractBufferedFile.readlines..Nrr`r`)rsplitr)rGrlinesrrrr readlinesCs   zAbstractBufferedFile.readlinescCs |j|S)N)ri)rGrrrr readinto1NszAbstractBufferedFile.readinto1cCsvt|ddrdS|jrdS|jdkr,d|_n@|js>|jdd|jdk rl|jj|j|jj|jj |jd|_dS)z=Close file Finalizes writes, discards cache Z _unclosableFNrT)ra) r!rIrrLrOr]rrer]rs)rGrrrcloseQs    zAbstractBufferedFile.closecCs|jdko|j S)zWhether opened for readingr)rrI)rGrrrreadablefszAbstractBufferedFile.readablecCs|jS)z'Whether is seekable (only in read mode))rz)rGrrrseekablejszAbstractBufferedFile.seekablecCs|jdko|j S)zWhether opened for writingrrE>rEr)rrI)rGrrrwritablenszAbstractBufferedFile.writablecCs|js|jdS)N)rIry)rGrrr__del__rszAbstractBufferedFile.__del__cCsdt|jj|jfS)Nz)rNrr5r])rGrrr__str__vszAbstractBufferedFile.__str__cCs|S)Nr)rGrrr __enter__{szAbstractBufferedFile.__enter__cGs |jdS)N)ry)rGrrrr__exit__~szAbstractBufferedFile.__exit__iiP)rrBTrCNN)r)F)Fr`)r`)rjN))r5r6r7r8rGrQrr<rJsetterrRrIrLrPrXrYrrrrr]r`r_rKrrirorprrrsrwrxryrzr{r|r}r~__repr__rrrrrrrsV H     *    r)#r loggingrr(rDrerrnorrrhashlibr callbacksrconfigrrrCr rcr utilsr r r rr getLoggerrcrrNrr:IOBaserrrrrs6       ;*