Name	Size	Mode	Actions
abc.cpython-36.pyc	1206	0644	editdlrm
csv.cpython-36.pyc	3996	0644	editdlrm
json.cpython-36.pyc	4597	0644	editdlrm
parquet.cpython-36.pyc	3319	0644	editdlrm
text.cpython-36.pyc	1493	0644	editdlrm
__init__.cpython-36.pyc	145	0644	editdlrm

3 <%Ega�@s�dd
lZdd
l
Z
ddlmZmZmZ
dd
lZddlmZm Z m Z mZ
ddlm Z
ddlmZ
ddlmZ
ddlmZmZ
d d lmZ
Gdd�de�ZGd d�d�Zd
S)�N)�BinaryIO�Optional�Union�)�Dataset�Features� NamedSplit�config)
�query_table)
�Json)
�logging)�NestedDataStructureLike�PathLike�
)
�AbstractDatasetReadercsFeZ
dZdeeeeeeee eed��f
dd� Z dd�Z�ZS) �JsonDatasetReaderNF)� path_or_paths�split�features� cache_dir�keep_in_memory�fieldcsZt�j
|
f
||||d
�|��

||_t|
t�r2|
n|j|
i
}
tf||
||d�|��
|_dS)N)rrrr)r� data_filesrr)�super�__init__r� isinstance�dictrr�builder)�selfrrrrrr�kwargs)
� __class__��:/usr/local/lib/python3.6/site-packages/datasets/io/json.pyrs 







zJsonDatasetReader.__init__c
CsJd}
d}d
}d}d}d}|jj
|
|||||d�
|jj|j||jd�}|S)NTF)�download_config� download_mode�ignore_verifications�try_from_hf_gcs� base_path�use_auth_token)rr%Z in_memory)rZdownload_and_prepareZ as_datasetrr)rr#r$r%r&r(r'�datasetr!r!r"�read's












zJsonDatasetReader.read)NNNFN) �__name__� __module__�__qualname__r rrrr�str�boolrr*� __classcell__r!r!)
r r"rs



.rc@sXeZ
dZd eeeefeeeed�dd�Z ed�
dd�Z dd �Zeed �dd�Zd
S)�JsonDatasetWriterN)r)�path_or_buf� batch_size�num_proccKsR|dk r |d
k
r td|�d��
�
|
|_
||_|r4|ntj|_||_d|_||_dS)Nr
z num_proc z must be an integer > 0.zutf-8) � ValueErrorr)r2r ZDEFAULT_MAX_BATCH_SIZEr3r4�encoding�to_json_kwargs)rr)r2r3r4r7r!r!r"r@s





zJsonDatasetWriter.__init__)
�returnc
 Cs�|jj
d
d�}
|jj
dd�}|jj
dd�}|jj
dd�}|dkrPtd|�d��
�
t|jtttjf�r�t j |jd |d�� }|jf|||d�|j��
}WdQRXn0|r�td|�d��
�
|jf|j||d�|j��
}|S)Nr2�orient�records�linesT�compression�infer�gzip�bz2�xzz&`datasets` currently does not support z compression�wb)
r<)�file_objr9r;zUThe compression parameter is not supported when writing to a buffer, but compression=z1 was passed. Please provide a local path instead.)Nr=r>r?r@)r7�pop�NotImplementedErrorrr2r.�bytes�osr�fsspec�open�_write)r�
_r9r;r<�buffer�writtenr!r!r"�writeRs






&


zJsonDatasetWriter.writecCsh|
\}}}}t|j
jt|||j�|j
jd
�}|j�jfd||d�|��
}|jd�
s\|d7}|j |j �
S)N)�table�key�indices)r2r9r;�
)r r)�data�slicer3Z_indicesZ to_pandas�to_json�endswith�encoder6)r�args�offsetr9r;r7�batch�json_strr!r!r"�_batch_jsongs






zJsonDatasetWriter._batch_json)rBr8cs
d
}|jdks|jdkrlx�t
jtd
t|j�
|j�dt
j�dd�D]$}|j|�
��f�
}||
j |�
7}qBWn�t|j�
|j}} t j|j�
�t} xlt
j| j|j��
�fdd�td
|| �D�
�|| r�|| dn|| dt
j�dd �D]}||
j |�
7}q�WWdQRX|S) z�Writes the pyarrow table as JSON lines to a binary file handle. Caller is responsible for opening and closing the handle. r
Nr�bazCreating json from Arrow format)�unit�disable�descc
sg|]}
|
�
��f�qSr!r!)�.0rX)r;r9r7r!r"� �sz,JsonDatasetWriter._write..)�totalr]r^r_) r4rZtqdm�range�lenr)r3Zis_progress_bar_enabledr[rM�multiprocessing�Pool�imap)rrBr9r;r7rLrXrZZnum_rowsr3�poolr!)r;r9r7r"rIts*













zJsonDatasetWriter._write)NN) r+r,r-rrrrr�intrrMr[rIr!r!r!r"r1?s
& r1)rerF�typingrrrrG�rrrr Z formattingr Zpackaged_modules.json.jsonr�utilsrZutils.typingr r�abcrrr1r!r!r!r"�
s






0