Name	Size	Mode	Actions
convert.cpython-36.pyc	6175	0644	editdlrm
datasets_cli.cpython-36.pyc	1549	0644	editdlrm
dummy_data.cpython-36.pyc	15745	0644	editdlrm
env.cpython-36.pyc	1688	0644	editdlrm
run_beam.cpython-36.pyc	4589	0644	editdlrm
test.cpython-36.pyc	5122	0644	editdlrm
__init__.cpython-36.pyc	778	0644	editdlrm

3 <%Eg��@s�dd
lZdd
l
Z
dd
lZddlmZmZ
ddlmZ
ddlm Z
dZ dZddd d ddgZd(d)d*d+d,d-d.d/d0d1d2gZ ed#�
d$d%�ZGd&d'�d'e�Zd
S)3�N)�ArgumentParser� Namespace)
�BaseDatasetsCLICommand)
� get_loggerz><<<<<<< This should probably be modified because it mentions: z======= >>>>>>> ZTextEncoderConfigZByteTextEncoderZSubwordTextEncoderZencoder_configZmaybe_build_from_corpusZ manual_dir� tfds\.core�datasets�tf\.io\.gfile\.GFile�open� tf\.([\w\d]+)�datasets.Value('\1')�tfds\.features\.Text\(\)�datasets.Value('string')�tfds\.features\.Text\(�datasets.Value('string'),�+features\s*=\s*tfds.features.FeaturesDict\(�features=datasets.Features(�tfds\.features\.FeaturesDict\(�dict(�The TensorFlow Datasets Authors�DThe TensorFlow Datasets Authors and the HuggingFace Datasets Authors�tfds\.� datasets.�dl_manager\.manual_dir�self.config.data_dir�self\.builder_config�self.config)
�argsc

Cst|j
|j�S)
zz Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint. Returns: ConvertCommand )�ConvertCommand� tfds_path�datasets_directory)
r�r �C/usr/local/lib/python3.6/site-packages/datasets/commands/convert.py�convert_command_factory*sr"c@s6eZ
dZeed
�
dd��
Zeed�dd�Zdd�Zd S) r)
�parserc
CsB|jd
dd�}
|
j
dtddd�
|
j
dtdd d�
|
jtd �

dS)z� Register this command to argparse so it's available for the datasets-cli Args: parser: Root parser to register command-specific arguments �convertzHConvert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.)
�helpz--tfds_pathTzQPath to a TensorFlow Datasets folder to convert or a single tfds file to convert.)�type�requiredr%z--datasets_directoryz(Path to the HuggingFace Datasets folder.)
�funcN)� add_parser�add_argument�str�set_defaultsr")r#Ztrain_parserr r r!�register_subcommand4s






z"ConvertCommand.register_subcommand)rrcGstd
�
|_
|
|_||_dS)Nzdatasets-cli/converting)r�_logger� _tfds_path�_datasets_directory)�selfrrrr r r!�__init__Ks

zConvertCommand.__init__c
sjtj
j|j�
rtj
j|j�
}
n&tj
j|j�
rWd�k�r�tjd��}|jdd�|jd �
jd!�
D�
�

d"|jd �
�d#�k�s�d$�k�s�d�k�r�td%�j ����
�
d&�k�s�d'�k�r�d} |j��

�
qPW| �sd(|k�rh|jdd�}tj
j ||��
tj
j �
|�} tj!�
dd)�
|j j d*�
���

|j"�
f
d+d,�|D�
�

n |j| �

|�r�|j| �

t| d-d d ��} | j#|�

WdQRX|j j d.| ���

q�Wx~|D]v}yDtj
j|�
}||jdd�}|j j d/|�d|���

t$j%||�
Wn*t&k �r4


|j j'd0|�d1��

YnX�q�W|�rfx"|D]}|j j(d2|�d3��

�qHWdS)4NzA--tfds_path is neither a directory nor a file. Please check path.zConverting datasets from z to zLooking at file r2�_testz.pyz Skipping filezutf-8)
�encodingFz!import tensorflow.compat.v2 as tfz @tfds.corezbuilder=selfz-import tensorflow_datasets.public_api as tfdszimport datasets zimport tensorflow�zfrom absl import loggingzfrom datasets import logging � getLoggerrc
3s|]}
|
�kV
qdS)
Nr )�.0Z expression)
�out_liner r!� �sz%ConvertCommand.run..Tc

s|�kS)
Nr )
�
e)
r8r r!��sz$ConvertCommand.run..�
Ztensorflow_datasetsz/from\stensorflow_datasets.*import\s([^\.\r\n]+)c
ss|]}
|
j�V
qdS)
N)
�strip)r7�impr r r!r9�s�
�
,zfrom . import ztf.ztfds.zError converting ZGeneratorBasedBuilderZBeamBasedBuilderZwmt)
�exist_okzAdding directory c
si|] }
�|
�qSr r )r7r>)
� output_dirr r!� �sz&ConvertCommand.run..�
wz Converted in zMoving z#Cannot find destination folder for z. Please copy manually.z!You need to manually update file z4 to remove configurations using 'TextEncoderConfig'.))�os�path�isdirr/�abspath�isfile�dirname� ValueErrorr0r.�info�listdir�basename�joinr � readlines�replace�any�TO_HIGHLIGHT�list�filter�append�HIGHLIGHT_MESSAGE_PREr+�HIGHLIGHT_MESSAGE_POST� TO_CONVERT�re�sub�match�extend�group�splitr=�makedirs�update� writelines�shutil�copy�KeyError�error�warning)r1Z abs_tfds_pathZabs_datasets_pathZutils_filesZwith_manual_updateZimports_to_builder_mapZ file_names�f_nameZ input_fileZoutput_file�
f�linesZ out_linesZ is_builderZneeds_manual_updateZtfds_imports�lineZ to_remove�pattern�replacementr\�dir_nameZ utils_fileZdest_folder� file_pathr )r8rBr!�runQs�









.







































 












zConvertCommand.runN) �__name__� __module__�__qualname__�staticmethodrr-r+r2rpr r r r!r3s

r)rr)rr )r r)rr )rr)rr)rr)rr)rr)rr)rr)rErZrc�argparserrZdatasets.commandsrZdatasets.utils.loggingrrWrXrSrYr"rr r r r!�
s4