Name	Size	Mode	Actions
arrow.cpython-36.pyc	7328	0644	editdlrm
cached.cpython-36.pyc	23030	0644	editdlrm
dask.cpython-36.pyc	4656	0644	editdlrm
dbfs.cpython-36.pyc	13351	0644	editdlrm
dvc.cpython-36.pyc	2622	0644	editdlrm
ftp.cpython-36.pyc	10444	0644	editdlrm
git.cpython-36.pyc	3750	0644	editdlrm
github.cpython-36.pyc	7404	0644	editdlrm
hdfs.cpython-36.pyc	7093	0644	editdlrm
http.cpython-36.pyc	20752	0644	editdlrm
jupyter.cpython-36.pyc	4131	0644	editdlrm
libarchive.cpython-36.pyc	6005	0644	editdlrm
local.cpython-36.pyc	11912	0644	editdlrm
memory.cpython-36.pyc	6697	0644	editdlrm
reference.cpython-36.pyc	16142	0644	editdlrm
sftp.cpython-36.pyc	6104	0644	editdlrm
smb.cpython-36.pyc	10591	0644	editdlrm
tar.cpython-36.pyc	4502	0644	editdlrm
webhdfs.cpython-36.pyc	14712	0644	editdlrm
zip.cpython-36.pyc	3276	0644	editdlrm
__init__.cpython-36.pyc	156	0644	editdlrm

3 8%Eg��@s�dd
lZdd
l
Z
dd
lZddlmZ
ddlmZ
ddlmZ
ddl m Z
ddlmZm Z
e ddd d ddd dddddddddddddddd dd ddddg�Gdd�de ��
ZGdd �d e�Zd
S)!�N)
�suppress)
�HadoopFileSystem)
�wrap_exceptions)
�AbstractFileSystem)�infer_storage_options�mirror_from�client�chmod�chown�user�df� disk_usage�download�driver�exists� extra_confZget_capacityZget_space_used�hostZis_open�kerb_ticketZstrip_protocol�mkdir�portZread_parquet�rm�stat�uploadcs�eZ
dZd
Zd#Zd$�f
dd � Zed d��
Zedd ��
Z dd�Z dd�Zed%dd�
�
Z edd��
Zdd�Zedd��
Zedd��
Zed&dd�
�
Zed'd!d"�
�
Z�ZS)(�PyArrowHDFSz�Adapted version of Arrow's HadoopFileSystem This is a very simple wrapper over the pyarrow.hdfs.HadoopFileSystem, which passes on all calls to the underlying class.�hdfs�file�defaultr
N�libhdfscsNt�j
f|�

t|
|||||d
��_tj��f
dd��
|
|||||f�_dS)a7 Parameters ---------- host: str Hostname, IP or "default" to try to read from Hadoop config port: int Port to connect on, or default from Hadoop config if 0 user: str or None If given, connect as this username kerb_ticket: str or None If given, use this ticket for authentication driver: 'libhdfs' or 'libhdfs3' Binary driver; libhdfs if the JNI library and default extra_conf: None or dict Passed on to HadoopFileSystem )rrrrrrc
s �jj
�S)
N)r�close�)
�selfr�E/usr/local/lib/python3.6/site-packages/fsspec/implementations/hdfs.py�[sz&PyArrowHDFS.__init__..N)�super�__init__rr�weakref�finalize�pars)r rrrrrr�kwargs)
� __class__)
r r!r$6s





zPyArrowHDFS.__init__c
CsXt|�
}
i}|
j
d
d�r$|
d
|d
<|
j
dd�r<|
d|d<|
j
dd�rT|
d|d<|S)Nr�usernamerr)r�get)�path�ops�outrrr!�_get_kwargs_from_urls_s







z!PyArrowHDFS._get_kwargs_from_urlscCs*t|
�
}|d
}
|
j
d�
r&|
dd�}
|
S)Nr,zfile:�)r� startswith)�clsr,r-rrr!�_strip_protocolks 


zPyArrowHDFS._strip_protocolcCs t|j
fS)
N)rr')r �protocolrrr!� __reduce_ex__us
zPyArrowHDFS.__reduce_ex__c


Cs|jj
�
dS)
N)rr)
r rrr!rxs
zPyArrowHDFS.closeTcs8�f
d
d��jj
|
dd�D�
}|r&|Sdd�|D�
SdS)Nc
sg|]}
�j|
�
�qSr)
� _adjust_entry)�.0�entry)
r rr!� ~sz"PyArrowHDFS.ls..T)
�detailc
Ssg|]}
|
d�qS)
�namer)r7r8rrr!r9�s)r�ls)r r,r:�listingr)
r r!r<{s
zPyArrowHDFS.lscCs|j|j
j|
�
�
S)
N)r6r�info)r r,rrr!r>�szPyArrowHDFS.infocCs^|
j�}d
|kr$d|kr$|d|d
<d|kr@d|kr@|d|d<d|krZ|j
|d�
|d<|S)N�type�kindr;r,)�copyr3)r Zoriginal_entryr8rrr!r6�s








zPyArrowHDFS._adjust_entrycKs�|j|
�
r|j
|�

dS|j|
�
��}d
j|j|�
dtjd�
��g�
}y6|j|d��}tj||�
WdQRX|j j ||�
Wn6tk r�


tt �
�
|j j|�

WdQRX�YnXWdQRXdS)N�
/z.tmp.��wb)�isdir�makedirs�open�join�_parent�secretsZ token_hex�shutil�copyfileobjr�mv� BaseExceptionr�FileNotFoundErrorr)r ZlpathZrpathr(ZlstreamZ tmp_fnameZrstreamrrr!�cp_file�s









zPyArrowHDFS.cp_filecCs|jj
|
�
S)
N)rr)r r,rrr!�rm_file�szPyArrowHDFS.rm_fileFcCs(|r|j|
�
rt
|
�
�
|jj|
d
d�S)NT)
Zcreate_parents)r�FileExistsErrorrr)r r,�exist_okrrr!rF�s
zPyArrowHDFS.makedirs�rbcKst||
|f|||d
�|��
S)a�
Parameters ---------- path: str Location of file; should start with '/' mode: str block_size: int Hadoop block size, e.g., 2**26 autocommit: True Transactions are not yet implemented for HDFS; errors if not True kwargs: dict or None Hadoop config parameters Returns ------- HDFSFile file-like instance )� block_size� autocommit� cache_options)
�HDFSFile)r r,�moderUrVrWr(rrr!�_open�s






zPyArrowHDFS._open)rr)rr
NNrN)
T)
F)rTNTN)�__name__� __module__�__qualname__�__doc__r4r$�staticmethodr/�classmethodr3r5rrr<r>r6rPrQrFrZ� __classcell__rr)
r)r!r s2%




" 
  



rc@sDeZ
dZd
Zddd�
Zddd �
Zd d�Zdd �Zdd�Zdd�Z dS)rXz`Wrapper around arrow's HdfsFile Allows seek beyond EOF and (eventually) commit/discard T� readaheadNc KsX|std
�
�
|
|_
||_||_||_|_|
jj|||f|�
|_|jj �rT|j �|_dS)Nz2HDFSFile cannot be opened with 'autocommit=False'.)�NotImplementedError�fsr,rY� blocksizerUrrG�fh�readable�size� seek_size) r rdr,rYrUrVZ cache_typerWr(rrr!r$�s







zHDFSFile.__init__r
cCs*|d
kr|j�rt
|
|j�}
|jj|
|�S)Nr
)rg�minrirf�seek)r �loc�whencerrr!rk�s


z HDFSFile.seekcCst|j
|
�S)
N)�getattrrf)r �itemrrr!�__getattr__

s
zHDFSFile.__getattr__cCst|j
|j|j|jffS)
N)rXrdr,rYrU)r r4rrr!r5
s
zHDFSFile.__reduce_ex__c


Cs|S)
Nr)
r rrr!� __enter__
s
zHDFSFile.__enter__c
Cs|j�
dS)
N)
r)r �exc_type�exc_val�exc_tbrrr!�__exit__
s
zHDFSFile.__exit__)TrbN)
r
) r[r\r]r^r$rkrpr5rqrurrrr!rX�s

 rX)rJrKr%� contextlibrZpyarrow.hdfsrZfsspec.implementations.arrowrZfsspec.specrZfsspec.utilsrrr�objectrXrrrr!�
sP
































0