/usr/local/lib/python3.6/site-packages/fsspec/__pycache__/
/usr/local/lib/python3.6/site-packages/fsspec/__pycache__
NameSizeModeActions
archive.cpython-36.pyc26790644editdlrm
asyn.cpython-36.pyc232500644editdlrm
caching.cpython-36.pyc133250644editdlrm
callbacks.cpython-36.pyc59270644editdlrm
compression.cpython-36.pyc46450644editdlrm
config.cpython-36.pyc27400644editdlrm
conftest.cpython-36.pyc13610644editdlrm
core.cpython-36.pyc202340644editdlrm
dircache.cpython-36.pyc32920644editdlrm
exceptions.cpython-36.pyc7330644editdlrm
fuse.cpython-36.pyc101100644editdlrm
gui.cpython-36.pyc142140644editdlrm
mapping.cpython-36.pyc82080644editdlrm
parquet.cpython-36.pyc111810644editdlrm
registry.cpython-36.pyc76210644editdlrm
spec.cpython-36.pyc522480644editdlrm
transaction.cpython-36.pyc31780644editdlrm
utils.cpython-36.pyc144030644editdlrm
_version.cpython-36.pyc4640644editdlrm
__init__.cpython-36.pyc13940644editdlrm
Edit: /usr/local/lib/python3.6/site-packages/fsspec/__pycache__/asyn.cpython-36.pyc (23250B)
3 8%Egj@sddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddlmZddlmZddlmZmZmZejd ZdddZ.de dddfddZ/d d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2gZ0Gd3d4d4eZ1d5d6Z2Gd7d8d8e3Z4d9d9e4dfd:d;Z5dS)?N)contextmanager) has_magic)_DEFAULT_CALLBACK)FSTimeoutError)AbstractFileSystem)PY36 is_exception other_pathsz_[^_]cst|r|nd}|dk r"tj||d}zBy|IdH|d<Wn*tk r`}z||d<WYdd}~XnXWd|jXdS)N)timeoutr)asynciowait_for Exceptionset)eventcororesultr exr5/usr/local/lib/python3.6/site-packages/fsspec/asyn.py_runners r)r c Os|r|nd}|dks|jr$tdyt}||kr.wrapper) functoolswraps)r#r(r)r)r#r(r sync_wrapperQsr,c csPtj}z6tjdkr6tjdkr6ttdr6tjtjdVWdtj|XdS)NntWindowsSelectorEventLoopPolicy)r-r.) r get_event_loop_policysys version_infoosnamehasattrset_event_loop_policyr0)Zoriginal_policyrrr_selector_policy`s    r8cCs4ttdrtjStj}|dkr,tdn|SdS)Nget_running_loopzno running event loop)r6r r9_get_running_loopr)r"rrrr9ps   r9cCsxtddkrptZtddkrfttjtd<WdQRXtjtdjdd}d|_|j |t d<WdQRXtdS)zdCreate or return the default fsspec IO loop The loop will be running on a separate thread. rNZfsspecIO)targetr5T) r"lockr8r new_event_looprThread run_foreverdaemonstartiothread)thrrrget_loop{s  rDccsPy t}Wntk r"d}YnXt}ztj||VWdtj|XdS)zTemporarily switch the current event loop to the fsspec's own loop, and then revert it back after the context gets terinated. N)r9rrDr _set_running_loop)Z original_loop fsspec_looprrrrFs    rFiFc Csddlm}|r"d|kr2|dSnd|kr2|dS|r:tStdkrFtSytjtj\}}Wnttt fk rttSX|tj krdS|dSdS)Nr)confZnofiles_gather_batch_sizeZgather_batch_sizerr.) Z fsspec.configrH_NOFILES_DEFAULT_BATCH_SIZEresource_DEFAULT_BATCH_SIZEZ getrlimitZ RLIMIT_NOFILE ImportError ValueError ResourceErrorZ RLIM_INFINITY)nofilesrHZ soft_limit_rrr_get_batch_sizes"   rRc s|dkrt|d}|d kr"t|}|dks.tg}xltdt||D]X}fdd||||D}tk rfdd|D|jtj|d|iIdHqDW|S) aSRun the given coroutines in chunks. Parameters ---------- coros: list of coroutines to run batch_size: int or None Number of coroutines to submit/wait on simultaneously. If -1, then it will not be any throttling. If None, it will be inferred from _get_batch_size() callback: fsspec.callbacks.Callback instance Gets a relative_update when each coroutine completes timeout: number or None If given, each coroutine times out after this time. Note that, since there are multiple batches, the total run time of this function will in general be longer return_exceptions: bool Same meaning as in asyncio.gather nofiles: bool If inferring the batch_size, does this operation involve local files? If yes, you normally expect smaller batches. N)rPrrcs g|]}tjtj|dqS))r )r Taskr ).0c)r rr sz(_run_coros_in_chunks..csg|]}|jfddqS)cs jddS)NZrelative_updater)call)rQ__)callbackrrsz1_run_coros_in_chunks...)add_done_callback)rTt)rYrrrVsreturn_exceptionsrI)rRlenAssertionErrorrangerextendr gather) coros batch_sizerYr r]rPresultsrAchunkr)rYr r_run_coros_in_chunkss    rg_ls _cat_file _get_file _put_file_rm_file_cp_file _pipe_file _expand_path_info_isfile_isdir_exists_walk_glob_find_du_size_mkdir _makedirscs@eZdZdZdZdZddddfdd Zedd Zd d Z dAd d Z ddZ dBddZ dCddZ ddZdDddZdEddZdFddZddZdedfdd Zd!d"Zdefd#d$Zd%d&Zd'd(Zd)d*ZdGd+d,Zd-d.Zd/d0ZdHd1d2ZdId3d4Zd5d6ZdJd7d8Z dKd9d:Z!dLd;d<Z"dMd=d>Z#dNd?d@Z$Z%S)OAsyncFileSystemakAsync file operations, default implementations Passes bulk operations to asyncio.gather for concurrent operation. Implementations that have concurrent batch operations and/or async methods should inherit from this class instead of AbstractFileSystem. Docstrings are copied from the un-underscored method in AbstractFileSystem, if not given. TFN) asynchronousr"rdcs@||_tj|_|s"|pt|_nd|_||_tj||dS)N) r|r4getpid_pidrD_looprdsuper__init__)r'r|r"rdr$r%) __class__rrr&s zAsyncFileSystem.__init__cCs|jtjkrtd|jS)NzThis class is not fork-safe)r~r4r}rr)r'rrrr"0szAsyncFileSystem.loopcstdS)N)r)r'pathr%rrrrl6szAsyncFileSystem._rm_filecsB|pj}j||dIdH}tfdd|D|ddIdHS)N) recursivecsg|]}j|fqSr)rl)rTp)r%r'rrrV>sz'AsyncFileSystem._rm..T)rdrP)rdrorg)r'rrrdr%r)r%r'r_rm9s  zAsyncFileSystem._rmcstdS)N)r)r'path1path2r%rrrrmCszAsyncFileSystem._cp_filec s|dkr|rd}n |dkrd}j|||dIdH}t||}|pFj}fddt||D} t| |dddIdH} x*tt| D]} |dkrt| trq| qWdS)Nignoreraise)maxdepthrcs g|]\}}j||fqSr)rm)rTp1p2)r%r'rrrVXsz)AsyncFileSystem._copy..T)rdr]rP) ror rdziprgfilterr rFileNotFoundError) r'rrron_errorrrdr%pathsrcrrr)r%r'r_copyFs   zAsyncFileSystem._copycsDt|tr||i}|pj}tfdd|jD|ddIdHS)Ncs g|]\}}j||fqSr)rn)rTkv)r%r'rrrVgsz)AsyncFileSystem._pipe..T)rdrP)rstrrdrgitems)r'rvaluerdr%r)r%r'r_pipebs  zAsyncFileSystem._pipecsd}d}|dk rR|dkrR|dkr0| }d}d}q^|pF|j|IdHd}||}n |dkr^d}|s|dk r|dkr|dk r|p|j|IdHd}||}n |dkrd}t|tr|d8}d||fS) z"Helper for "Range"-based _cat_fileNFrTsizerz bytes=%s-%s)rprint)r'urlrAendrsuffrrr_process_limitsls*   zAsyncFileSystem._process_limitscstdS)N)r)r'rrArr%rrrriszAsyncFileSystem._cat_filerc sj||dIdH}fdd|D}|p0j}t||dddIdH}dkrhttt|d} | rh| t|dkst|ts|d j |krfd d t ||DS|d SdS) N)rcsg|]}j|fqSr)ri)rTr)r%r'rrrVsz(AsyncFileSystem._cat..T)rdrPr]rFrrcs(i|] \}}dkst| r||qS)Zomit)r )rTrr)rrr sz(AsyncFileSystem._cat..) rordrgnextrr r^rlist_strip_protocolr) r'rrrrdr%rrcoutrr)r%rr'r_cats    zAsyncFileSystem._catcs|dk r tt|tstt|ts2|gt|}t|tsJ|gt|}t|t|ksjt|t|krntfddt|||D}|pj}t||ddIdHS)Ncs*g|]"\}}}j|f||dqS))rAr)ri)rTrse)r%r'rrrVsz/AsyncFileSystem._cat_ranges..T)rdrP) rrr TypeErrorr^rNrrdrg)r'rZstartsendsZmax_gaprdr%rcr)r%r'r _cat_rangess      zAsyncFileSystem._cat_rangescstdS)N)r)r'lpathrpathr%rrrrkszAsyncFileSystem._put_filecs(ddlm}m}j|}t|tr,||}|} | j||d} t| |t|to^j|IdHd} dd| Dfdd t | | D} fd d t | | D} t j fd d | DIdH|pȈj }g}|j d t| x4| D],\}}|j||||jj||f|qWt|||d IdHS)aCopy file(s) from local. Copies a specific file or tree of files (if recursive=True). If rpath ends with a "/", it will be assumed to be a directory, and target files will go within. The put_file method will be called concurrently on a batch of files. The batch_size option can configure the amount of futures that can be executed at the same time. If it is -1, then all the files will be uploaded concurrently. The default can be set for this instance by passing "batch_size" in the constructor, or for all instances by setting the "gather_batch_size" key in ``fsspec.config.conf``, falling back to 1/8th of the system limit . r)LocalFileSystemmake_path_posix)rN)existscSsi|]}tjj||qSr)r4risdir)rTlrrrrsz(AsyncFileSystem._put..csg|]\}}|r|qSrr)rTrr)is_dirrrrVsz(AsyncFileSystem._put..cs g|]\}}|s||fqSrr)rTrr)rrrrVscsg|]}j|ddqS)T)exist_ok)rz)rTd)r'rrrVsset_size)rdrY)Zimplementations.localrrrrrZ expand_pathr rrrr rbrdrWr^branchappendrkrg)r'rrrrYrdr%rrfslpathsrpathsZrdirsZ file_pairsrcZlfilerfiler)rr'r_puts(  " zAsyncFileSystem._putcstdS)N)r)r'rrr%rrrrjszAsyncFileSystem._get_filec sddlm}|j|}||}|j||dIdH}t||}dd|D|jd|j} g} |jdt|x:t ||D],\}}|j |||| j |j ||f|qvWt | | |d IdHS) aCopy file(s) to local. Copies a specific file or tree of files (if recursive=True). If lpath ends with a "/", it will be assumed to be a directory, and target files will go within. Can submit a list of paths, which may be glob-patterns and will be expanded. The get_file method will be called concurrently on a batch of files. The batch_size option can configure the amount of futures that can be executed at the same time. If it is -1, then all the files will be uploaded concurrently. The default can be set for this instance by passing "batch_size" in the constructor, or for all instances by setting the "gather_batch_size" key in ``fsspec.config.conf``, falling back to 1/8th of the system limit . r)r)rNcSs"g|]}tjtjj|ddqS)T)r)r4makedirsrdirname)rTZlprrrrVsz(AsyncFileSystem._get..rdr)rdrY)Zfsspec.implementations.localrrror poprdZ lazy_callr^rrrrjrg) r'rrrrYr%rrrrdrcrrr_gets   zAsyncFileSystem._getc s(y|j|IdHddkSdSdS)NtypefileF)rp)r'rrrrrqszAsyncFileSystem._isfilec s2y|j|IdHddkStk r,dSXdS)Nr directoryF)rpIOError)r'rrrrrrszAsyncFileSystem._isdircs|j|IdHjddS)Nr)rpget)r'rrrrrx szAsyncFileSystem._sizecs*|pj}tfdd|D|dIdHS)Ncsg|]}j|qSr)rx)rTr)r'rrrV&sz*AsyncFileSystem._sizes..)rd)rdrg)r'rrdr)r'r_sizes#s zAsyncFileSystem._sizesc s.y|j|IdHdStk r(dSXdS)NTF)rpr)r'rrrrrs)s zAsyncFileSystem._existscstdS)N)r)r'rr%rrrrp0szAsyncFileSystem._infocstdS)N)r)r'rdetailr%rrrrh3szAsyncFileSystem._lscKs|j|}i}i}i}|jdd}y |j|fddi|IdH}Wn6ttfk rx|rh|iifVn |ggfVdSXxj|D]b} | djd} | jddd } | ddkr| |kr| || <| || <q| |kr| |d <q| || <qW|r|||fVn|t|t|fV|dk r,|d8}|dkr,dSx\|D]T} xL|j| f||d |2dHy 3dH} Wn t k sxX| VqTYWq2WdS) NrFTr5/rrrr)rrrI) rrrhrrrstriprsplitrrtStopAsyncIteration)r'rrr%Z full_dirsdirsfilesrlistinginfopathnamer5rrQrrrrt6s>           8zAsyncFileSystem._walkcshddl}|jd}|j|}|jddkr4|jdnt|}|jddkrT|jdnt|}|jddkrt|jdnt|}t|||}|jdd} t|s|} d} |r|d 7}n<|j|IdHr| s|gS||j |IdHiSn | sgSiSnd|d|krN|d|j d} |d| d} d |kr2dn|| ddj dd} n,d } d |kr`dn||ddj dd} |j | f| d d d |IdHd|j ddj ddj ddj ddj ddj ddj ddj ddj ddj dd j d!d"jdj ddd|jd#d$|jd%d&|jj d$d'fd(d)tD} | r\| St| SdS)*Nrr*?[rFrz/*z**rT)rwithdirsr^\z\\.z\.+z\+z//(z\()z\)|z\|z\^$z\${z\{}z\}z[*]{2}z =PLACEHOLDER=z[*]z[^/]*z.*cs.i|]&}j|jddjdr||qS)z//r)matchreplacer)rTr)allpathspatternrrrsz)AsyncFileSystem._glob..)reendswithrfindr^minrrrsrprindexcountrvrrsubcompilesortedr)r'rr%rrZindstarZindquesZindbraceindrrootdepthZind2rr)rrrrucsJ        *(r  zAsyncFileSystem._globcsbi}xD|j|fd|i|IdHD]$}|j|IdH}|d||d<q"W|rZt|jS|SdS)Nrrr5)rvrpsumvalues)r'rtotalrr%sizesfrrrrrws" zAsyncFileSystem._duc s|j|}t|jdd}xn|j||fddi|2dHy3dH\}}}Wn tk sX|rf|j|jdd|jDq:YW r|j|IdHri|<t} |s| Sfdd| DSdS)NrFTcSsi|]\}}||dqS)r5r)rTr5rrrrrsz)AsyncFileSystem._find..csi|]}||qSrr)rTr5)rrrrs) rdictrrtrupdaterrqr) r'rrrr%rrQrrnamesr)rrrvs  < &zAsyncFileSystem._findcst|tr"j|g||IdH}n|s*|n|d}t}fdd|D}x|D]}t|rtj|IdH}||O}|rP|tjt|||dIdHO}qPn&|rtj||ddIdH}||O}||krP|dksj|IdHrP|j |qPW|st |tt |S)Nrcsg|]}j|qSr)r)rTr)r'rrrVsz0AsyncFileSystem._expand_path..)rrT)rrF) rrrorrrurrvrsaddrr)r'rrrrrbitZrecr)r'rros,   zAsyncFileSystem._expand_pathcsdS)Nr)r'rZcreate_parentsr%rrrryszAsyncFileSystem._mkdircsdS)Nr)r'rrrrrrzszAsyncFileSystem._makedirs)FN)FNNN)NN)NN)FrN)NN)N)T)N)TN)NF)FN)T)F)&__name__ __module__ __qualname____doc__Z async_implZdisable_throttlingrpropertyr"rlrrmrrrrirrrkrrrjrrqrrrxrrsrprhrtrurwrvroryrz __classcell__rr)rrr{sJ        ,"   -K   r{cCsddlm}xtttD]}|jds*q|dd}tj|rtj t ||d}t t ||ddd}|t ||dk}|r|rt t |||d }t ||||j st t ||dd d|_ qWdS) a,Populate sync and async methods for obj For each method will create a sync version if the name refers to an async method (coroutine) and there is no override in the child class; will create an async method for the corresponding sync method if there is no implementation. Uses the methods specified in - async_methods: the set that an implementation is expected to provide - default_async_methods: that can be derived from their sync version in AbstractFileSystem - AsyncFileSystem: async-specific default coroutines r)rrQrNF__func__r)r(r)Zfsspecr async_methodsdirr{ startswithprivaterinspectiscoroutinefunctiongetattrr,setattrr)r(rmethodZsmethodZiscoZunsyncZ is_defaultZmthrrrmirror_sync_methodss     r c@s eZdZdS)FSSpecCoroutineCancelN)rrrrrrrr sr Tc sddltrtdddtjjtdD}|r@dd|Dfdd|D}|rxp|D]hj}jtj j j |tj j jfdd|Dyj j |Wq^|k rYq^Xq^W|S)NrzDo not call this on Py 3.6cSsg|]}|js|qSr)done)rTr\rrrrVsz'_dump_running_tasks..cSsg|] }|jqSr) print_stack)rTtaskrrrrVsc sLg|]D}|jjj|jjjj|jjjj|jjjj|jjr@|nddqS)N)localsr firstlineZlinelostackr)_corocr_framef_localsf_code co_filenameco_firstlinenof_lineno format_stack)rTr) traceback with_taskrrrV scsg|]}|dqS)rr)rTcb)r\rrrV0s)rrrr tasks all_tasksr" _callbackscancelfuturesFuture set_exceptionrthrow)Zprintoutr!excrrrZcbsr)r\rrr_dump_running_taskss(   r')N)N)F)6r Zasyncio.eventsr*rr4rr2r contextlibrglobr callbacksr exceptionsrspecrutilsrr r rrreventsr:rr9r&rBr"Lockr<r,r8rDrFrKrMOSErrorrOerrorZ ResourceErorrLrJrRrgrr{r rr r'rrrrs        &        4]