/usr/local/lib/python3.6/site-packages/fsspec/implementations/__pycache__
Edit: /usr/local/lib/python3.6/site-packages/fsspec/implementations/__pycache__/reference.cpython-36.pyc (16142B)
3
8%EgH @ s d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlZy d dl Z
W n e k
rd d dl
Z
Y nX ddl m
Z
mZ ddlmZ ddlmZmZ ddlmZ ddlmZ ejd ZG d
d d e
Zd d
ZdS ) N) lru_cache )AsyncFileSystemsync)_DEFAULT_CALLBACK)
filesystemopen)
get_mapper)AbstractFileSystemzfsspec.referencec s e Zd ZdZdZd; fdd Zedd Zd d
Zd
ddZd?ddZdd Zd@d d Zd d ZdAd d!Zd"d# Zd$d% Zd&d' ZdBd)d*ZdCd+d,Zd-d. Zd/d0 Zd1d2 Z dDd3d4Z dE fd5d6 Z d7d8 Z d9d: Z Z!S )FReferenceFileSystema View byte ranges of some other file as a file system
Initial version: single file system target, which must support
async, and must allow start and end args in _cat_file. Later versions
may allow multiple arbitrary URLs for the targets.
This FileSystem is read-only. It is designed to be used with async
targets (for now). This FileSystem only allows whole-file access, no
``open``. We do not get original file details from the target FS.
Configuration is by passing a dict of references at init, or a URL to
a JSON file containing the same; this dict
can also contain concrete data for some set of paths.
Reference dict format:
{path0: bytes_data, path1: (target_url, offset, size)}
https://github.com/fsspec/kerchunk/blob/main/README.md
referenceNTc
s t jf d| i|
|| _d| _| | _|
| _i | _t|drJ|j }nt |t
r|rdd|i}ni }t f |pv|pvi |}| dks|j drddl
}ddl}d| _t|f|}|j| jd d
kst jd | _ jjd d| _|j fd
ddD jd| _n| dks$|j drddl}d| _t|df|Z}|j|}|jd d
ks`tt j |jd | _t j |jjd d| _|j dd| _W dQ R X n.t|df|}t j d| |j }W dQ R X n|}| jr| j! n | j"|| |dk r|| _#dS |dkrTx@| jj$ D ]2}t%|r2| }t&j'j(|\}}|r |}P q W |dkrxZ| j)j$ D ]L}t%|r~| }t |t*rj|d rjt&j'j(|d \}}|rj|}P qjW |dkr|}t+|fd| i|pi | _#dS )a
Parameters
----------
fo : dict or str
The set of references to use for this instance, with a structure as above.
If str, will use fsspec.open, in conjunction with ref_storage_args to
open and parse JSON at this location.
target : str
For any references having target_url as None, this is the default file
target to use
ref_storage_args : dict
If references is a str, use these kwargs for loading the JSON file
target_protocol : str
Used for loading the reference file, if it is a path. If None, protocol
will be derived from the given path
target_options : dict
Extra FS options for loading the reference file, if given as a path
remote_protocol : str
The protocol of the filesystem on which the references will be evaluated
(unless fs is provided). If not given, will be derived from the first
URL that has a protocol in the templates or in the references, in that
order.
remote_options : dict
kwargs to go with remote_protocol
fs : file system instance
Directly provide a file system, if you want to configure it beforehand. This
takes precedence over target_protocol/target_options
template_overrides : dict
Swap out any templates in the references file with these - useful for
testing.
ref_type : "json" | "parquet" | "zarr"
If None, guessed from URL suffix, defaulting to JSON. Ignored if fo
is not a string.
simple_templates: bool
Whether templates can be processed with simple replace (True) or if
jinja is needed (False, much slower). All reference sets produced by
``kerchunk`` are simple in this sense, but the spec allows for complex.
kwargs : passed to parent class
loopFreadprotocolzarrr NTversion templatesgenc s i | ]} | d d |qS )N ).0k)zr J/usr/local/lib/python3.6/site-packages/fsspec/implementations/reference.py
s z0ReferenceFileSystem.__init__..keydataurloffsetsizeZparquetrbz[])indexzRead reference from URL %s)r r r r r ),super__init__target dataframetemplate_overridessimple_templatesr hasattrr
isinstancestrdictendswithZpandasr r Z
open_groupattrsAssertionErrorgetr Z DataFrameZ set_indexdfZ fastparquetr Z ParquetFileZkey_value_metadatajsonloadsZ to_pandasloggerinfo_process_dataframe_process_referencesfsvaluescallablefsspeccoreZsplit_protocol
referenceslistr )selffor$ Zref_storage_argsZtarget_protocolZtarget_optionsZremote_protocolZremote_optionsr7 r& r' r
Zref_typekwargstextextraZdicpdr mfpfpfrefr _) __class__)r r r# / s 8
z ReferenceFileSystem.__init__c C s | j jr| j jS | jS )N)r7
async_implr
_loop)r> r r r r
s zReferenceFileSystem.loopc C s$ | j |}tjd| | jrN| jj| }|d r>|d }qX|dddg }n
| j| }t|trj|j }t|t
rtjd| d |j drt j
|d d }|d d fS t|d
krtjd| d |d }d }d }n.|\}}}tjd| d
| d| || }|d kr| j}|||fS )Nzcat: r r r r z Reference: z , type bytess base64: r z , whole filer z , offset z, size )_strip_protocolr3 debugr% r0 locr< r) r* encodebytes
startswithbase64 b64decodelenr$ )r> pathpartr startendr r r r _cat_common s4
z ReferenceFileSystem._cat_commonc sF | j |\}}}t|tr&||| S | jj|||dI d H || S )N)rY rZ )r[ r) rR r7 _cat_file)r> rW rY rZ r@ part_or_urlstart0end0r r r r\ s
z ReferenceFileSystem._cat_filec K s@ | j |\}}}t|tr&||| S | jj|||d|| S )N)rY rZ )r[ r) rR r7 cat_file)r> rW rY rZ r@ r] r^ r_ r r r r` s
z ReferenceFileSystem.cat_filec K s || j |< dS )z2Temporarily add binary data or reference as a fileN)r< )r> rW valuerI r r r pipe_file s z ReferenceFileSystem.pipe_filec sL | j |rtj|ddS | j|I d H }t|d}|j| W d Q R X d S )NT)exist_okwb)isdirosmakedirsr\ r write)r> rpathlpathr@ r rF r r r _get_file s
z ReferenceFileSystem._get_filec K sf | j |rtj|ddS | j|f|}|jdt| t|d}|j| W d Q R X |jdt| d S )NT)rc Zset_sizerd Zabsolute_update)re rf rg r` Z lazy_callrV r rh )r> ri rj callbackr@ r rF r r r get_file s
z ReferenceFileSystem.get_fileFc K s: | j jr t| j| j|||f|S tj| ||fd|i|S )N recursive)r7 rK r r
_getr
r/ )r> ri rj rn r@ r r r r/ s zReferenceFileSystem.getc sf j jr tjj||f S t|trV|s>tdd |D rBt fdd|D S t j
|S d S )Nc s s | ]}d |kV qdS )*Nr )r pr r r s z*ReferenceFileSystem.cat..c s i | ]}t j|f |qS r )r
r` )r rq )r@ r> r r r s z+ReferenceFileSystem.cat..) r7 rK r r
Z_catr) r= anyNotImplementedErrorr
r` )r> rW rn r@ r )r@ r> r cat s
zReferenceFileSystem.catc sH j j td fdd} jr< jd j| jd< j d S )Ni c sL dd l }d| krH jr4| jddjddjf jS |j| jf jS | S )Nr z{{{z}}})jinja2r' replaceformatr Templaterender)r rx )r> r r
_render_jinja s
z=ReferenceFileSystem._process_dataframe.._render_jinjar )_process_templatesr r r0 map_dircache_from_items)r> r} r )r> r r5 s
z&ReferenceFileSystem._process_dataframec C sj t |ttfrtj|}|jdd }|d kr8| j| n&|dkrP| j||d ntd| | j d S )Nr r )r& z Unknown reference spec version: )
r) r* rR r1 r2 r/ _process_references0_process_references1
ValueErrorr )r> r< r& versr r r r6 s
z'ReferenceFileSystem._process_referencesc C s d|krt |}|| _dS )z&Make reference dict for Spec Version 0Zzarr_consolidated_formatN)
_unmodel_hdf5r< )r> r< r r r r * s z(ReferenceFileSystem._process_references0c sX j sjrHy dd l W n, tk
rF } ztd|W Y d d }~X nX i _j|jdi td fdd}x|jdi j D ]\}}t
|t r|j drt
j|d d j|< |j|< q|d }d
|kr
j r|jd
d jd d
jf j}n||}t|dkr |gn||d |d gj|< qW jjj|jdg d S )Nr z(Reference Spec Version 1 requires jinja2r i c s j | jf jS )N)r{ r| r )u)rx r> r r r} : s z?ReferenceFileSystem._process_references1.._render_jinjarefszbase64:rM z{{rv z}}rw r r r )r' r rx ImportErrorr r< r~ r/ r itemsr) r* rS rT rU ry rz rV update _process_gen)r> r< r& er} r vr r )rx r> r r 1 s,
0z(ReferenceFileSystem._process_references1c sh dd l i | _| jd k r$|j| j x>|j D ]2\}}d|krV|f fdd | j|< q.|| j|< q.W d S )Nr z{{c s j | jf |S )N)r{ r| )tempr@ )rx r r Y s z8ReferenceFileSystem._process_templates..)rx r r& r r )r> tmpr r r )rx r r~ Q s
z&ReferenceFileSystem._process_templatesc s dd l }i }x |D ]}dd |d j D fddtj j D }x|D ]}|j|d jf || j}|j|d jf || j}d |krd
|krt|j|d jf || j} t|j|d
jf || j}
|| |
g||< qRd |kd
|kA r
t d qR|g||< qRW qW |S ) Nr c S s@ i | ]8\}}t |tr|n t|jd d|d |jdd|qS )rY r stopstepr )r) r= ranger/ )r r r r r r r d s z4ReferenceFileSystem._process_gen..Z
dimensionsc 3 s | ]}t t j |V qd S )N)r+ zipkeys)r r8 ) dimensionr r rr k s z3ReferenceFileSystem._process_gen..r r r lengthz^Both 'offset' and 'length' are required for a reference generator entry if either is provided.)
rx r itertoolsproductr8 r{ r| r intr ) r> Zgensrx outr Zproductsprr r r r r )r r r _ s*
z ReferenceFileSystem._process_genc C s. dg i| _ | jr | jj }n
| jj }x |D ]\}}| jr\|d rRt|d }q|d }n4t|tt frtt|}n t|dkrd }n
|\}}}d|kr|j
ddd nd}|}x\|o|| j kr g | j |< | j j d|kr|j
ddd ndg j |ddd | j
|}qW | j | j |d |d q.W d S )
N r r r /r directory)nametyper file)Zdircacher% r0 Ziterrowsr< r rV r) rR r* rsplit
setdefaultappend_parent) r> itrW rX r rI rY parZpar0r r r r s.
z(ReferenceFileSystem._dircache_from_itemsr c K s |dkr t | j|}tj|S )Nr )rt r` ioBytesIO)r> rW mode
block_sizeZ
cache_optionsr@ r r r r r s
zReferenceFileSystem.openc K s6 | j |}| j|}|d kr t|r(|S dd |D S )Nc S s g | ] }|d qS )r r )r or r r
s z*ReferenceFileSystem.ls..)rN _ls_from_cacheFileNotFoundError)r> rW detailr@ r r r r ls s
zReferenceFileSystem.lsc K s( y| j |d k S tk
r" dS X d S )NF)r r )r> rW r@ r r r exists s zReferenceFileSystem.existsc C s | j |o| j|d dkS )Nr r )r r4 )r> rW r r r re s zReferenceFileSystem.isdirc C s | j |o| j|d dkS )Nr r )r r4 )r> rW r r r isfile s zReferenceFileSystem.isfilec s | j ||f|S )N)r )r> rW r r@ r r r _ls s zReferenceFileSystem._lsc sN |r t j f||d|S rD| j t fdd| jD S t| jS )N)maxdepthwithdirsc 3 s | ]}|j r|V qd S )N)rS )r r )rW r r rr s z+ReferenceFileSystem.find..)r" findrN sortedr< )r> rW r r r@ )rJ )rW r r s
zReferenceFileSystem.findc s6 | j d} fdd|D }|s. dddS |d S )NTc s g | ]}|d kr|qS )r r )r r )rW r r r s z,ReferenceFileSystem.info..r r )r r r )r )r> rW r@ r Zout0r )rW r r4 s
zReferenceFileSystem.infoc s
| j |S )N)r4 )r> rW r@ r r r _info s zReferenceFileSystem._info) NNNNNNNNTNN)NN)NN)F)F)N)N)r NN)T)T)NF)"__name__
__module__ __qualname____doc__r r# propertyr
r[ r\ r` rb rk r rm r/ ru r5 r6 r r r~ r r r r r re r r r r4 r
__classcell__r r )rJ r r sL u !
#
r c C s ddl }i }x| d j D ]\}}|jdr|jdd }|jd|}|rf|jd|d d |d }xF|j D ]&\}}||d
|d
|d f||< qpW qtj|j ||< qW |S ) z'Special JSON format from HDF5 prototyper Nmetadataz .zchunkstoresourceuriz#https://([^.]+)\.s3\.amazonaws\.comzhttps://z.s3.amazonaws.comr r ) rer r, popfindallry r1 dumpsrQ ) r< r rH r ra r matchr r r r r r s
$r ) rT r r loggingrf functoolsr Z fsspec.corer: Zujsonr1 r Zasynr r callbacksr r; r r mappingr specr
getLoggerr3 r r r r r r s*
=