/usr/local/lib/python3.6/site-packages/fsspec/__pycache__
Edit: /usr/local/lib/python3.6/site-packages/fsspec/__pycache__/utils.cpython-36.pyc (14403B)
3
8%EgA @ s& d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm Z d dl
m Z d dl m
Z
d7Zejd8k Zd9d d Zd:d
dZi Zdd Zdd Zdd Zd;ddZdd Zdd Zd d Zd d Zd
d4d5Z%dS )? N)contextmanager)partial)md5)urlsplit c C sZ t jd| s t jd| dkr&d| dS t| }|jp6d}|jrRdj|j|jg}n|j}|dkr|t jd|}|r|d|j }|dkr|| dS ||d}|jr"|jj d d d j d
d d |d< |d kr|d |d |d< n |d |d< |j
r|j
|d< |j r|j |d< |j r"|j |d< |j
r4|j
|d< |jrF|j|d< |rVt|| |S ) a Infer storage options from URL path and merge it with existing storage
options.
Parameters
----------
urlpath: str or unicode
Either local absolute file path or URL (hdfs://namenode:8020/file.csv)
inherit_storage_options: dict (optional)
Its contents will get merged with the inferred information from the
given path
Returns
-------
Storage options dict.
Examples
--------
>>> infer_storage_options('/mnt/datasets/test.csv') # doctest: +SKIP
{"protocol": "file", "path", "/mnt/datasets/test.csv"}
>>> infer_storage_options(
... 'hdfs://username:pwd@node:123/mnt/datasets/test.csv?q=1',
... inherit_storage_options={'extra': 'value'},
... ) # doctest: +SKIP
{"protocol": "hdfs", "username": "username", "password": "pwd",
"host": "node", "port": 123, "path": "/mnt/datasets/test.csv",
"url_query": "q=1", "extra": "value"}
z^[a-zA-Z]:[\\/]z^[a-zA-Z0-9]+://Nfile)protocolpath#z^/([a-zA-Z])[:|]([\\/].*)$z%s:%shttphttps@ :r hosts3s3agcsgsr
portusernamepasswordZ url_queryZ url_fragment)r r )r r r r )rematchr schemefragmentjoinr
groupsnetlocrsplitr r r queryupdate_storage_options)ZurlpathZinherit_storage_optionsZ parsed_pathr r
Z windows_pathoptions r( 6/usr/local/lib/python3.6/site-packages/fsspec/utils.pyinfer_storage_options sB
"
r* c C sX |si }t | t |@ }|rJx,|D ]$}| j||j|kr"td| q"W | j| d S )Nz;Collision between inferred and specified storage option:
%s)setgetKeyErrorupdate)r' Z inheritedZ
collisionsZ collisionr( r( r) r& b s
r& c C s. t jj| d jdj }|tkr*t| S dS )a Infer compression, if available, from filename.
Infer a named compression type, if registered and available, from filename
extension. This includes builtin (gz, bz2, zip) compressions, as well as
optional compressions. See fsspec.compression.register_compression.
r .Nr )osr
splitextstriplower compressions)filename extensionr( r( r) infer_compressiont s r7 c s, | d7 } t tjtj| fdd}|S )a Returns a function that receives a single integer
and returns it as a string padded by enough zero characters
to align with maximum possible integer
>>> name_f = build_name_function(57)
>>> name_f(7)
'07'
>>> name_f(31)
'31'
>>> build_name_function(1000)(42)
'0042'
>>> build_name_function(999)(42)
'042'
>>> build_name_function(0)(0)
'0'
g:0yE>c s t | j S )N)strzfill)i)
pad_lengthr( r)
name_function s z*build_name_function..name_function)intmathceillog10)Zmax_intr< r( )r; r) build_name_function s rA c C s | j dkrdS d}x| j|}|s(dS |r4|| n|}yL||krr|j|}| j| j t|| t| dS t||k rdS W n ttfk
r Y nX |t| d }qW dS )a Seek current file to file start, file end, or byte after delimiter seq.
Seeks file to next chunk delimiter, where chunks are defined on file start,
a delimiting sequence, and file end. Use file.tell() to see location afterwards.
Note that file start is a valid split, so must be at offset > 0 to seek for
delimiter.
Parameters
----------
file: a file
delimiter: bytes
a delimiter like ``b'\n'`` or message sentinel, matching file .read() type
blocksize: int
Number of bytes to read from the file at once.
Returns
-------
Returns True if a delimiter was found, False if at file start or end.
r FNT)tellreadindexseeklenOSError
ValueError)r delimiter blocksizelastcurrentfullr: r( r( r) seek_delimiter s$
" rN Fc
C s |r| j | t| |d}|dkr*| j S | j }||| 8 }| j || t| |d}| j }|rt|rt|t|8 }|r|r|t|8 }|}|| }| j | | j|} | S )a Read a block of bytes from a file
Parameters
----------
f: File
Open file
offset: int
Byte offset to start read
length: int
Number of bytes to read, read through end of file if None
delimiter: bytes (optional)
Ensure reading starts and stops at delimiter bytestring
split_before: bool (optional)
Start/stop read *before* delimiter bytestring.
If using the ``delimiter=`` keyword argument we ensure that the read
starts and stops at delimiter boundaries that follow the locations
``offset`` and ``offset + length``. If ``offset`` is zero then we
start at zero, regardless of delimiter. The bytestring returned WILL
include the terminating delimiter string.
Examples
--------
>>> from io import BytesIO # doctest: +SKIP
>>> f = BytesIO(b'Alice, 100\nBob, 200\nCharlie, 300') # doctest: +SKIP
>>> read_block(f, 0, 13) # doctest: +SKIP
b'Alice, 100\nBo'
>>> read_block(f, 0, 13, delimiter=b'\n') # doctest: +SKIP
b'Alice, 100\nBob, 200\n'
>>> read_block(f, 10, 10, delimiter=b'\n') # doctest: +SKIP
b'Bob, 200\nCharlie, 300'
r Ni i )rE rN rC rB rF )
foffsetlengthrI Z split_beforeZfound_start_delimstartZfound_end_delimendbr( r( r)
read_block s& %
rV c O sP |r| |f7 } yt t| j j S tk
rJ t t| j ddj S X dS )zDeterministic token
(modified from dask.base)
>>> tokenize([1, 2, '3'])
'9d71491b50023b06fc76928e6eddb952'
>>> tokenize('Hello') == tokenize('Hello')
True
F)usedforsecurityN)r r8 encode hexdigestrH )argskwargsr( r( r) tokenize s
r\ c C sL t | tr| S t| dr | j S t | tjr4t| S t| drD| jS | S dS )aD Attempt to convert a path-like object to a string.
Parameters
----------
filepath: object to be converted
Returns
-------
filepath_str: maybe a string version of the object
Notes
-----
Objects supporting the fspath protocol (Python 3.6+) are coerced
according to its __fspath__ method.
For backwards compatibility with older Python version, pathlib.Path
objects are specially coerced.
Any other object is passed through unchanged, which includes bytes,
strings, buffers, or anything else that's not even path-like.
__fspath__r
N)
isinstancer8 hasattrr] pathlibPathr
)filepathr( r( r) stringify_path# s
rc c C s | ||}|j |S )N)Z_determine_worker)clsrZ r[ instr( r( r)
make_instanceE s
rf c sr dd | D t dd D }d}x.t|D ]" t fddD }|s.P q.W |7 djd d S ) z;For a list of paths, find the shortest prefix common to allc S s g | ]}|j d qS )/)split).0pr( r( r)
M s z!common_prefix..c s s | ]}t |V qd S )N)rF )ri rj r( r( r) N s z common_prefix..r c 3 s" | ]}| d kV qdS )r Nr( )ri rj )r: partsr( r) rl Q s rg N)minrangeallr! )pathsZlmaxrT r( )r: rm r)
common_prefixK s rr c s t tr|pjd}jdt| dkr`t| |rJ jddd fdd| D q|rjdd | d jdd gqgnt| tkstS )a In bulk file operations, construct a new file tree from a list of files
Parameters
----------
paths: list of str
The input file tree
path2: str or list of str
Root to construct the new list in. If this is already a list of str, we just
assert it has the right number of elements.
is_dir: bool (optional)
For the special case where the input in one element, whether to regard the value
as the target path, or as a directory to put a file path within. If None, a
directory is inferred if the path ends in '/'
exists: bool (optional)
For a str destination, it is already exists (and is a dir), files should
end up inside.
Returns
-------
list of str
rg r r c s g | ]}|j d qS )r )replace)ri rj )cppath2r( r) rk u s z other_paths..r )r^ r8 endswithrstriprF rr r$ AssertionError)rq ru is_direxistsr( )rt ru r) other_pathsX s
$r{ c C s
t | tS )N)r^
BaseException)objr( r( r) is_exception s r~ c C s& t jd| d}t|dkr"|d S dS )Nz (\:\:|\://)r r r )r rh rF )urlrm r( r( r) get_protocol s r c C s> ddl m} yt|t| ddS ttfk
r8 dS X dS )z*Can the given URL be used with open_local?r )get_filesystem_class
local_fileFN)Zfsspecr getattrr rH ImportError)r
r r( r( r) can_be_local s
r c C s | t jkr$t j| }t|dr$|jS t jdkrTyddl}|jj| jS Y qvX n"yddl }|j| jS Y nX yddl
}|j | }|jS t t
fk
r dS X dS )a For given package name, try to find the version without importing it
Import and package.__version__ is still the backup here, so an import
*might* happen.
Returns either the version string, or None if the package
or the version was not readily found.
__version__r r N)r r )sysmodulesr_ r version_infoZimportlib.metadatametadata distributionversionimportlib_metadata importlib
import_moduler AttributeError)namemodr r r( r( r) "get_package_version_without_import s*
r DEBUGTc C sh | d kr|d krt d| p$tj|} tj }tjd}|j| |rP| jj | j| | j | | S )Nz+Provide either logger object or logger namezD%(asctime)s - %(name)s - %(levelname)s - %(funcName)s -- %(message)s)
rH logging getLogger
StreamHandler Formatter setFormatterhandlersclear
addHandlersetLevel)loggerZ logger_namelevelr handle formatterr( r( r)
setup_logging s
r c C sT t |jtr*| j|jr | S |jd | S | jt|jr>| S |jd d | S d S )Nz://r )r^ r r8
startswithtuple)r fsr( r( r) _unstrip_protocol s r c s fdd fdd}|S )zqMirror attributes and methods from the given
origin_name attribute of the instance to the
decorated classc s t | }t || S )N)r )methodselforigin) origin_namer( r)
origin_getter s
z"mirror_from..origin_getterc s, x& D ] }t |}t| |t| qW | S )N)r setattrproperty)rd r Zwrapped_method)methodsr r( r) wrapper s
z mirror_from..wrapperr( )r r r r( )r r r r) mirror_from s r c c s
| V d S )Nr( )r} r( r( r) nullcontext s r c
C sz t | tstt |ts&|gt| }t |ts>|gt| }t|t| ks^t|t| krbtt|dkrx| ||fS |rdd ttt| || D \} }}| rp| dd }|dd }|dd }xtdt| D ]} | | | | d ks*|| |d |ks*|dk rV|| |d |krV|j| | |j|| |j|| q|| |d< qW |||fS | ||fS )a} Merge adjacent byte-offset ranges when the inter-range
gap is <= `max_gap`, and when the merged byte range does not
exceed `max_block` (if specified). By default, this function
will re-order the input paths and byte ranges to ensure sorted
order. If the user can guarantee that the inputs are already
sorted, passing `sort=False` will skip the re-ordering.
r c S s g | ] }t |qS r( )list)ri vr( r( r) rk s z'merge_offset_ranges..Nr r r ) r^ r TypeErrorrF rH zipsortedro append)
rq ZstartsZendsZmax_gapZ max_blocksortZ new_pathsZ
new_startsZnew_endsr: r( r( r) merge_offset_ranges s4
$
r i i P )r r
)N)N)NF)NF)NNr T)r NT)&r r> r0 r` r r
contextlibr functoolsr hashlibr urllib.parser ZDEFAULT_BLOCK_SIZEr ZPY36r* r& r4 r7 rA rN rV r\ rc rf rr r{ r~ r r r r r r r r r( r( r( r) s@
R
1
A"
( $