/usr/local/lib/python3.6/site-packages/fsspec/implementations/__pycache__
Edit: /usr/local/lib/python3.6/site-packages/fsspec/implementations/__pycache__/arrow.cpython-36.pyc (7328B)
3
8%Egj @ s d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl m
Z
d dl m Z m
Z
mZ dd ZG dd d e
Ze
d
d d d
ddddgG dd dejZG dd deZdS ) N)suppress)wraps)AbstractFileSystem)infer_storage_options mirror_fromstringify_pathc s t fdd}|S )Nc sh y
| |S t k
rb } z<|js$ |j^}} t|trPd|krPttj||n W Y d d }~X nX d S )Nzdoes not exist)OSErrorargs
isinstancestrFileNotFoundErrorerrnoENOENT)r kwargs exceptionmessage)func F/usr/local/lib/python3.6/site-packages/fsspec/implementations/arrow.pywrapper s
z wrap_exceptions.
.wrapper)r )r r r )r r wrap_exceptions
s r c s e Zd ZdZdZ fddZedd Zd#dd Zd
d Z d d
Z
dd Z e dd Z
e dd ZeZe dd Ze d$ddZe d%ddZe d&d d Ze d'd d Ze d!d" Z ZS )(ArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem.
Parameters
----------
fs : pyarrow.fs.FileSystem
/c s || _ t jf | d S )N)fssuper__init__)selfr r ) __class__r r r * s zArrowFSWrapper.__init__c C s$ t |}d|kr |jd\}}}|S )Nz://)r partition)clspath_r r r _strip_protocol. s z ArrowFSWrapper._strip_protocolFc sD ddl m} fdd jj||D }|r2|S dd |D S d S )Nr ) FileSelectorc s g | ]} j |qS r ) _make_entry).0entry)r r r
: s z%ArrowFSWrapper.ls..c S s g | ] }|d qS )namer )r% r& r r r r' @ s )
pyarrow.fsr# r
get_file_info)r r Zdetailr r# entriesr )r r ls6 s
zArrowFSWrapper.lsc K s$ | j |}| jj|g\}| j|S )N)r" r r* r$ )r r r infor r r r- B s
zArrowFSWrapper.infoc C s6 | j |}y| j| W n tk
r, dS X dS d S )NFT)r" r- r )r r r r r existsG s
zArrowFSWrapper.existsc C sn ddl m} |j|jkr d}n<|j|jkr0d}n*|j|jkrVttjt j
tj|j nd}|j |j ||j
dS )Nr )FileType directoryfileother)r( sizetypemtime)r) r/ r4 Z DirectoryFileZNotFoundr r
r osstrerrorr r3 r5 )r r- r/ kindr r r r$ P s zArrowFSWrapper._make_entryc K s | j |jd}| j |jd}| j|d}dj| j|dtjd g}y6| j|d}tj || W d Q R X | j
j || W n6 t k
r t
t | j
j| W d Q R X Y nX W d Q R X d S )Nr rbz.tmp. wb)r" rstrip_openjoin_parentsecretsZ token_hexopenshutil copyfileobjr move
BaseExceptionr r delete_file)r path1path2r ZlstreamZ tmp_fnameZrstreamr r r cp_filec s
zArrowFSWrapper.cp_filec K s2 | j |jd}| j |jd}| jj|| d S )Nr )r" r= r rE )r rH rI r r r r mvs s zArrowFSWrapper.mvc C s | j |}| jj| d S )N)r" r rG )r r r r r rm_file{ s
zArrowFSWrapper.rm_fileNc C sF | j |jd}| j|r6|r,| jj| qBtdn | jj| d S )Nr z0Can't delete directories without recursive=False)r" r= isdirr
delete_dir
ValueErrorrG )r r recursiveZmaxdepthr r r rm s
zArrowFSWrapper.rmr: c K sN |dkr| j j|}n$|dkr,| j j|}ntd|t| ||||f|S )Nr: r< z'unsupported mode for Arrow filesystem: )r Zopen_input_streamZopen_output_streamrO ArrowFile)r r mode
block_sizer streamr r r r> s zArrowFSWrapper._openTc K s2 | j |}|r | j|dd n| jj|dd d S )NT)exist_okF)rP )r" makedirsr
create_dir)r r Zcreate_parentsr r r r mkdir s
zArrowFSWrapper.mkdirc C s | j |}| jj|dd d S )NT)rP )r" r rX )r r rV r r r rW s
zArrowFSWrapper.makedirsc C s | j |}| jj| d S )N)r" r rN )r r r r r rmdir s
zArrowFSWrapper.rmdir)F)FN)r: N)T)F)__name__
__module__ __qualname____doc__Z root_markerr classmethodr" r, r- r. r$ r rJ rK Zmv_filerL rQ r> rY rW rZ
__classcell__r r )r r r s*
r rU readseektellwritereadablewritableclosec @ s& e Zd ZdddZdd Zdd ZdS ) rR Nc K s. || _ || _|| _|| _| | _| _|| _d S )N)r rS r rU blocksizerT r )r r rU r rS rT r r r r r s zArrowFile.__init__c C s | S )Nr )r r r r __enter__ s zArrowFile.__enter__c G s | j S )N)rg )r r r r r __exit__ s zArrowFile.__exit__)N)r[ r\ r] r ri rj r r r r rR s
rR c s2 e Zd ZdZdZd
fdd Zedd Z ZS ) HadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem
to connect it's interface with fsspecZhdfsdefaultr Nc s8 ddl m} ||||||d}t jf d|i| dS )a
Parameters
----------
host: str
Hostname, IP or "default" to try to read from Hadoop config
port: int
Port to connect on, or default from Hadoop config if 0
user: str or None
If given, connect as this username
kerb_ticket: str or None
If given, use this ticket for authentication
extra_conf: None or dict
Passed on to HadoopFileSystem
r )rk )hostportuser kerb_ticket
extra_confr N)r) rk r r ) r rm rn ro rp rq r rk r )r r r r s zHadoopFileSystem.__init__c C sX t | }i }|jdd r$|d |d< |jdd r<|d |d< |jdd rT|d |d< |S )Nrm usernamero rn )r get)r opsoutr r r _get_kwargs_from_urls s z&HadoopFileSystem._get_kwargs_from_urls)rl r NNN) r[ r\ r] r^ protocolr staticmethodrv r` r r )r r rk s rk )r
ior7 rA rC
contextlibr functoolsr Z fsspec.specr Z fsspec.utilsr r r r r IOBaserR rk r r r r s