/usr/local/lib/python3.6/site-packages/fsspec/implementations/__pycache__/
/usr/local/lib/python3.6/site-packages/fsspec/implementations/__pycache__
NameSizeModeActions
arrow.cpython-36.pyc73280644editdlrm
cached.cpython-36.pyc230300644editdlrm
dask.cpython-36.pyc46560644editdlrm
dbfs.cpython-36.pyc133510644editdlrm
dvc.cpython-36.pyc26220644editdlrm
ftp.cpython-36.pyc104440644editdlrm
git.cpython-36.pyc37500644editdlrm
github.cpython-36.pyc74040644editdlrm
hdfs.cpython-36.pyc70930644editdlrm
http.cpython-36.pyc207520644editdlrm
jupyter.cpython-36.pyc41310644editdlrm
libarchive.cpython-36.pyc60050644editdlrm
local.cpython-36.pyc119120644editdlrm
memory.cpython-36.pyc66970644editdlrm
reference.cpython-36.pyc161420644editdlrm
sftp.cpython-36.pyc61040644editdlrm
smb.cpython-36.pyc105910644editdlrm
tar.cpython-36.pyc45020644editdlrm
webhdfs.cpython-36.pyc147120644editdlrm
zip.cpython-36.pyc32760644editdlrm
__init__.cpython-36.pyc1560644editdlrm
Edit: /usr/local/lib/python3.6/site-packages/fsspec/implementations/__pycache__/http.cpython-36.pyc (20752B)
3 8%Egd@sXddlmZmZmZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl Z ddl Z ddlmZmZmZddlmZddlmZddlmZdd lmZmZmZd d lmZejd Zejd Z ej!dZ"ddZ#GdddeZ$GdddeZ%d#ddZ&ejdZ'ddZ(GdddeZ)d$ddZ*d%dd Z+d&d!d"Z,ee,Z-dS)')absolute_importdivisionprint_functionN)copy)urlparse)AsyncFileSystemsync sync_wrapper)_DEFAULT_CALLBACK)FSTimeoutError)AbstractBufferedFile)DEFAULT_BLOCK_SIZE nullcontexttokenize)AllBytesz6<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P[^"']+)z-(?Phttp[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)z fsspec.httpcs tjf|S)N)aiohttpZ ClientSession)kwargsrE/usr/local/lib/python3.6/site-packages/fsspec/implementations/http.py get_clientsrc seZdZdZdZdddddddddef fdd Zed d Zd d Z e d dZ e fddZ d0ddZ d1ddZeeZddZd2ddZd4efddZd6edfdd Zd!d"Zd#d$Zd7d&d'Zd(d)Zd*d+Zd,d-Zd.d/ZZS)8HTTPFileSystema2 Simple File-System for fetching data via HTTP(S) ``ls()`` is implemented by loading the parent page and doing a regex match on the result. If simple_link=True, anything of the form "http(s)://server.com/stuff?thing=other"; otherwise only links within HTML href tags will be used. /TNbytesFc  stj|f||d| |dk r&|nt|_||_||_||_||_| pJi|_| |_ | |_ d|_ t | } | j dd|_| j dd| j dd| j dd| |_ |st|j|jdS)a NB: if this is called async, you must await set_client Parameters ---------- block_size: int Blocks to read bytes; if 0, will default to raw requests file-like objects instead of HTTPFile instances simple_links: bool If True, will consider both HTML tags and anything that looks like a URL; if False, will consider only the former. same_scheme: True When doing ls/glob, if this is True, only consider paths that have http/https matching the input URLs. size_policy: this argument is deprecated client_kwargs: dict Passed to aiohttp.ClientSession, see https://docs.aiohttp.org/en/stable/client_reference.html For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}`` get_client: Callable[..., aiohttp.ClientSession] A callable which takes keyword arguments and constructs an aiohttp.ClientSession. It's state will be managed by the HTTPFileSystem class. storage_options: key-value Any other parameters passed on to requests cache_type, cache_options: defaults used in open ) asynchronousloopNuse_listings_cacheFZlistings_expiry_timeZ max_pathsZskip_instance_cache)super__init__r block_size simple_links same_schema cache_type cache_options client_kwargsrr_sessionrpoprrr set_session) selfr rZ same_scheme size_policyr"r#rrr$rZstorage_optionsZrequest_options) __class__rrr,s$)    zHTTPFileSystem.__init__c Cs^|dk r>|jr>yt||jdddSttfk r<YnXt|dd}|dk rZ|jdS)Ng?)timeoutZ _connector) is_runningrclose TimeoutErrorr getattr_close)rsessionZ connectorrrr close_sessionns zHTTPFileSystem.close_sessioncsL|jdkrF|jfd|ji|jIdH|_|jsFtj||j|j|j|jS)Nr)r%rrr$rweakreffinalizer2)r(rrrr'{s   zHTTPFileSystem.set_sessioncCs|S)z-For HTTP, we always want to keep the full URLr)clspathrrr_strip_protocolszHTTPFileSystem._strip_protocolcs tj|}t|dkr|SdS)N)r_parentlen)r5r6par)r*rrr:s  zHTTPFileSystem._parentc s|jj}|j|tj||jIdH}|j|f|j4IdH }|j|||jIdH}WdQIdHRX|j rt j |ddt j |D}nddt j |D}t } t|} x|D]} t| tr| d} | jdrt| dkr| jd| j| } | jdrf|jr4| j|jddr4| j| n0| jddj|jddjddr| j| q| dkr| jdj|jd| jdgqW| r|jdr|j|jdd d IdH} |rd d| DStt| S| S)NcSsg|] }|dqS)rr).0urrr sz+HTTPFileSystem._ls_real..cSsg|] }|dqS)rr)r=r>rrrr?srz://httphttps..../F)detailcSs&g|]}|d|jdrdnddqS)Nr directoryfile)namesizetype)endswith)r=r>rrrr?s)rCrD)rrupdateloggerdebugr'get_raise_not_found_for_statustextr ex2findallexsetr isinstancetuple startswithr;schemenetlocr!rstripaddreplacejoinlstriprK_ls_reallistsorted) r(urlrErkwr1rrQlinksoutpartslrrrr`s@           $ zHTTPFileSystem._ls_realcsF|jr||jkr|j|}n&|j|fd|i|IdH}||j|<|S)NrE)rZdircacher`)r(rcrErrgrrr_lss   zHTTPFileSystem._lscCs|jdkrt||jdS)zU Raises FileNotFoundError for 404s, otherwise uses raise_for_status. iN)statusFileNotFoundErrorraise_for_status)r(responsercrrrrPs z*HTTPFileSystem._raise_not_found_for_statusc s|jj}|j|tj||dk s.|dk rj||kr:dS|jdij}|j|||IdH|d<||d<|jIdH}|j|f|4IdH }|j IdH} |j ||WdQIdHRX| S)NheadersRange) rrrLrMrNr&Z_process_limitsr'rOreadrP) r(rcstartendrrdrpr1rergrrr _cat_files   zHTTPFileSystem._cat_filerc s|jj}|j|tj||jIdH}|j|f|j4IdH}yt|jd} Wnt t fk rtd} YnX|j | |j ||t |d>} d} x2| r|jj|IdH} | j| |jt| qWWdQRXWdQIdHRXdS)Nzcontent-lengthwbT)rrrLrMrNr'rOintrp ValueErrorKeyErrorset_sizerPopencontentrrwriterelative_updater;) r(rpathlpath chunk_sizecallbackrrdr1rerIfdchunkrrr _get_files"        zHTTPFileSystem._get_filepostc sfdd}|jj}|j||jIdH} |j}|dkrPtd|t| |} | |fd|i|4IdH} |j| |WdQIdHRXdS)Nc sttjrt}d}ntd}d}|l}|rPj|jdd|jdnjt|dd|j}x&|r|Vj t ||j}qnWWdQRXdS)NFrbTrrrI) rVioIOBaserr}r|seekr/rrrr;)contextZuse_seekfr)rrrrr gen_chunkss    z,HTTPFileSystem._put_file..gen_chunksrputz.method has to be either 'post' or 'put', not: data)rr)rrrLr'lowerrzr/rP) r(rrrrmethodrrrdr1methrespr)rrrr _put_files      zHTTPFileSystem._put_filecs|jj}|j|yTtj||jIdH}|j|f|IdH}|4IdH |jdkSQIdHRXWntj t j fk rdSXdS)NiF) rrrLrMrNr'rOrkrequests HTTPErrorrZ ClientError)r(r6rrdr1rerrr_exists)s   zHTTPFileSystem._existscs|j|f|IdHS)N)r)r(r6rrrr_isfile5szHTTPFileSystem._isfilerc Ks|dkr t|dk r|n|j}|jj} |j| d<| j||pP|j|f|d}t|j|j } |r|rt ||f| ||||p|j |p|j |jd| St ||f||j| d| SdS)aMake a file-like object Parameters ---------- path: str Full URL with protocol mode: string must be "rb" block_size: int or None Bytes to download in one request; use instance value if None. If zero, will return a streaming Requests file-like instance. kwargs: key-value Any other parameters, passed to requests calls rNrrI)r1rmoderIr"r#r)rrr1)NotImplementedErrorrrrrrLinforrr'HTTPFiler"r#HTTPStreamFile) r(r6rrZ autocommitr"r#rIrrdr1rrr_open8s,   zHTTPFileSystem._opencCst||j|jS)z;Unique identifier; assume HTTP files are static, unchanging)rrprotocol)r(rcrrrukeykszHTTPFileSystem.ukeycsi}|jIdH}xd D]}y:|jt|f||d|j|IdH|jddk rTPWqtk r}z&|dkrzt||tjt |WYdd}~XqXqW|dd|ddiS) aHGet info of URL Tries to access location via HEAD, and then GET methods, but does not fetch the data. It is possible that the server does not supply any size information, in which case size will be given as None (and certain operations on the corresponding file will not work). NheadrO)r)r1rI)rHrIrJrG)rrO) r'rL _file_inforrO ExceptionrlrMrNstr)r(rcrrr1policyexcrrr_infoos$   $zHTTPFileSystem._infoc s>ddl}|jd}|j|}|jddkr4|jdnt|}|jddkrT|jdnt|}t||}|jdd}t|s|} d} |r|d 7}n<|j|IdHr|s|gS||j |IdHiSn |sgSiSnd|d|kr,|d|j d} |d| d} d |krdn|| ddj dd} n,d } d |kr>dn||ddj dd} |j | f| d d d |IdHd|j ddj ddj ddj ddj ddj ddj ddj ddj ddj dd j d!d"jdd|jd#d$|jd%d&|jj d$d'fd(d)tD} |r2| St| SdS)*a Find files by glob-matching. This implementation is idntical to the one in AbstractFileSystem, but "?" is not considered as a character for globbing, because it is so common in URLs, often identifying the "query" part. rNr*[rEFr@z/*z**r9T)ZmaxdepthZwithdirsrE^\z\\.z\.+z\+z//(z\()z\)|z\|z\^$z\${z\{}z\}z[*]{2}z =PLACEHOLDER=z[*]z[^/]*z.*cs.i|]&}j|jddjdr||qS)z//r)matchr]r[)r=p)allpathspatternrr sz(HTTPFileSystem._glob..)rerKr7findr;minr& has_magicrrrindexcount_findr]r[subcompilerbra) r(r6rrZendsZindstarZindbraceindrErootdepthZind2rgr)rrr_globsH       *(j  zHTTPFileSystem._globc s2yt|j|IdHSttfk r,dSXdS)NF)boolrjrlrz)r(r6rrr_isdirszHTTPFileSystem._isdir)T)T)NNiiPiiP)rNNNNN)__name__ __module__ __qualname____doc__seprr staticmethodr2r' classmethodr7r:r`rjr ZlsrPrur rrrrrrrrr __classcell__rr)r*rr sJ7   0   (  +"PrcsXeZdZdZdfdd Zdfd d Zd d ZeeZd dZ ee Z ddZ Z S)ra A file-like object pointing to a remove HTTP(S) resource Supports only reading, with read-ahead of a predermined block-size. In the case that the server does not supply the filesize, only reading of the complete file in one go is supported. Parameters ---------- url: str Full URL of the remote resource, including the protocol session: requests.Session or None All calls will be made within this session, to avoid restarting connections where the server allows this block_size: int or None The amount of read-ahead to do, in bytes. Default is 5MB, or the value configured for the FileSystem creating this file size: None or int If given, this is the size of the file in bytes, and we don't attempt to call the server to find the value. kwargs: all other key-values are passed to requests calls. NrrFc sZ|dkrtd| |_||_||_||dd|_tjf||||||d| | |_dS)NrzFile mode not supportedrG)rHrIrJ)fsr6rrr"r#)rrrcr1detailsrrr) r(rrcr1rrr"r#rIrrr)r*rrrszHTTPFile.__init__r@csl|dkr2|jdkr2|jdk o&|j|jk r2|j|jdkrN|dkr`|jnt|j|j|}tj|S)a5Read bytes from file Parameters ---------- length: int Read up to this many bytes. If negative, read all content to end of file. If the server has not supplied the filesize, attempting to read only part of the data will raise a ValueError. rN)locrI blocksize _fetch_allrrrr)r(length)r*rrrr!s   z HTTPFile.readc stjd|t|jts|jj|jf|jIdH}|4IdH<|j |j IdH}tt |dd|d|_t ||_ WdQIdHRXdS)zRead whole file in one shot, without caching This is only called when position is still at zero, and read() is called without a byte-count. zFetch all for N)rIZfetcherrr) rMrNrVcacherr1rOrcrrmrrr;rI)r(rergrrrasync_fetch_all8s zHTTPFile.async_fetch_allc sdtjd|d|d||jj}|jdij}d||df|d<tj|jd|d|jj|jfd|i|Id H}|4Id H|jd krd S|j |jd kr|j Id H}nt |j jd d||kr|j Id H}nfd}g}xF|j j dId H}|r4|j||t|7}|||kr6PqPqWd j|d ||}|SQId HRXd S)a3Download a block of data The expectation is that the server returns only the requested bytes, with HTTP code 206. If this is not the case, we first check the headers, and then stream the output - if the data size is bigger than we requested, an exception is raised. zFetch range for z: -rpz bytes=%i-%ir@rqz : NirozContent-Lengthrrrwi)rMrNrrr&rcr1rOrkrmrrryrpr~appendr;r^) r(rsrtrrprergZclrrrrasync_fetch_rangeKs4      zHTTPFile.async_fetch_rangecCs"t|j|j|j|j|jj|jffS)N)reopenrrcrrrrHrI)r(rrr __reduce__uszHTTPFile.__reduce__)NNrrNNNF)r) rrrrrrrrr rrZ _fetch_rangerrrr)r*rrs(rcCs|j|||||dS)N)rrr"rI)r})rrcrrr"rIrrrrsrz([*[])cCstj|}|dk S)N) magic_checksearch)srrrrrs rcsTeZdZdfdd ZddZddd ZeeZd d Zfd d Z ddZ Z S)rrNc svjdd__|_|_|dkr,tdd_tjf||ddfdd}t j|_ dS) NrFr)rHrInone)rr6rr"cs,jjfjIdH}jj||S)N)r1rO __aenter__rrP)re)rr(rcrrcorsz$HTTPStreamFile.__init__..cor) r&rrcrr1rzrrrrre)r(rrcrrr1rr)r*)rr(rcrrs zHTTPStreamFile.__init__cOs tddS)NzCannot seek streaming HTTP file)rz)r(argsrrrrrszHTTPStreamFile.seekr@cs*|jjj|IdH}|jt|7_|S)N)rer~rrrr;)r(numrgrrr_readszHTTPStreamFile._readcs|jjdS)N)rer-)r(rrrr0szHTTPStreamFile._closecs tj|j|jtjdS)N)asynciorun_coroutine_threadsafer0rrr-)r()r*rrr-szHTTPStreamFile.closecCst|j|j|j|j|jjffS)N)rrrcrrrrH)r(rrrrszHTTPStreamFile.__reduce__)rNNr)r) rrrrrrr rrr0r-rrrr)r*rrs  rc s|j}|jdij}d||df|d<|j|fd|i|IdH}|j|4IdH|jIdH}WdQIdHRX|rt|d} | j|| j|WdQRXn|SdS)Nrpz bytes=%i-%ir@rqzrb+)rr&rOrmrrr}rr) r1rcrsrtrGrrprergrrrr get_ranges  rrc s8tjd||j}|jdd}|jdij}d|d<||d<i}|dkrl|j|fd|i|IdH}n2|d kr|j|fd|i|IdH}n td ||4IdH~|jd |jkrt |jd |d <n&d |jkrt |jd j dd|d <x*dD]"}|jj|r|j|||<qWWdQIdHRX|S)zCall HEAD on the server to get details about the file (size/checksum etc.) Default operation is to explicitly allow redirects and use encoding 'identity' (no compression) to get the true size of the target. zRetrieve file size for %sallow_redirectsTrpidentityzAccept-EncodingrNrOz+size_policy must be "head" or "get", got %szContent-LengthrIz Content-Rangerr@ETag Content-MD5Digest)rrr) rMrNrr&rOr TypeErrorrmrprysplit) rcr1r)rarrrreZchecksum_fieldrrrrs,     $rcs<|dkrtIdH}t|f|d|i|IdH}|jdS)Nr1rI)rrrO)rcr1rrrrrr _file_sizes r)N)N)r)N). __future__rrrrrloggingrr3r urllib.parserrrZ fsspec.asynrrr Zfsspec.callbacksr Zfsspec.exceptionsr Z fsspec.specr Z fsspec.utilsr rrZcachingrrrTrR getLoggerrMrrrrrrrrrr file_sizerrrrsB         L  '  '