/usr/local/lib/python3.6/site-packages/datasets/utils/__pycache__
Edit: /usr/local/lib/python3.6/site-packages/datasets/utils/__pycache__/metadata.cpython-36.pyc (17940B)
3
<%Eg6T @ s d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d dl m
Z
d dl m Z m
Z
mZmZmZmZmZmZmZ yd dl mZ W n ek
r dd ZY nX yd dljZW n ek
r d dlZY nX d dlZd d
l mZ d Z e d e Z ej e!Z"e#ee e#f d
ddZ$e#ee e#f d
ddZ%e$d\Z&Z'e%d\Z(Z)e$d\Z*Z+e$d\Z,Z-e$d\Z.Z/e$d\Z0Z1dddge Z2Z3G dd d ej4Z5e
ee# d d d Z6e
eee#ee# f d d d!Z7eee# ee# f Z8d6eee# ee#ee# f f ee# e#e#ee
e ge9f e8d"d#d$Z:e ed%d&d'Z;e
ee> Z?eG d-d. d.Z@d/d0 ee@D e@_Ae!d1krd d2lBmCZC eCd3d4ZDeDjEd5 eDjF ZGe
eGjHZHe@jIeHZJeJjK dS )7 N)Counter) dataclassfields)Path) AnyCallableClassVarDictListOptionalTupleTypeUnion)get_argsc C s | j S )N)__args__)tp r A/usr/local/lib/python3.6/site-packages/datasets/utils/metadata.pyr s r ) resourceszDhttps://github.com/huggingface/datasets/tree/main/src/datasets/utils/)resourcereturnc C s$ t jt| }tj|t d| fS )Nz /resources/)
pkg_resources read_textr jsonloads BASE_REF_URL)r contentr r r load_json_resource$ s r c C s0 t jt| }dd |j D }|t d| fS )Nc S s6 i | ].}d |kr|j d d j |j d d j qS ) r r )splitstrip).0liner r r
+ s z%load_tsv_licenses..z /resources/)r r r
splitlinesr )r r licensesr r r load_tsv_licenses) s r( zlanguages.jsonzstandard_licenses.tsvz
tasks.jsonz
creators.jsonzsize_categories.jsonzmultilingualities.jsonoriginalextendedz extended\|.*c s& e Zd Zdd Zd fdd Z ZS )NoDuplicateSafeLoaderc sR fdd|j D }dd |D }t| fdd D }|rNtd| d S )Nc s g | ]\}} j | qS r )Zconstructed_objects)r# Zkey_node_)selfr r
@ s zRNoDuplicateSafeLoader._check_no_duplicates_on_constructed_node..c S s" g | ]}t |trt|n|qS r )
isinstancelisttuple)r# keyr r r r. A s c s g | ]} | d kr|qS )r r )r# r2 )counterr r r. C s zGot duplicate yaml keys: )valuer TypeError)r- nodekeysZduplicate_keysr )r3 r- r (_check_no_duplicates_on_constructed_node? s z>NoDuplicateSafeLoader._check_no_duplicates_on_constructed_nodeFc s t j||d}| j| |S )N)deep)superconstruct_mappingr8 )r- r6 r9 mapping) __class__r r r; G s
z'NoDuplicateSafeLoader.construct_mapping)F)__name__
__module__ __qualname__r8 r;
__classcell__r r )r= r r+ > s r+ )pathr c
C sn t | dd}dd |D }W d Q R X |d dkrjd|dd krjdj|d|dd jdd }|S d S ) Nzutf-8)encodingc S s g | ]}|j d qS )
)rstrip)r# r$ r r r r. O s z*yaml_block_from_readme..r z---r rD )openjoinindex)rB Z readme_filer Z yamlblockr r r yaml_block_from_readmeM s $rI c C s. t | d}|dkrdS tj|tdp(t }|S )zNLoads a dataset's metadata from the dataset card (REAMDE.md), as a Python dict)rB N)Loader)rI yamlloadr+ dict)rB Z
yaml_blockZ metada_dictr r r metadata_dict_from_readmeX s
rN )itemsreference_valuesnameurl escape_validation_predicate_fnr c s t jddj d t| trT d k r@ fdd| D }qfdd| D }nhg } d k rxZ| j D ] \}}| fdd|D 7 }qjW n,x*| j D ] \}}|fdd|D 7 }qW t|d krg | d
| d | fS | d fS ) Nz^(?:|z)$c s( g | ] }j | r |d kr|qS )F)match)r# v)rS rP r r r. o s z$tagset_validator..c s g | ]} j |s|qS r )rU )r# rV )rP r r r. r s c s( g | ] }j | r |d kr|qS )F)rU )r# rV )rS rP r r r. y s c s g | ]} j |s|qS r )rU )r# rV )rP r r r. } s r z are not registered tags for 'z', reference at )recompilerG r/ r0 rO len)rO rP rQ rR rS Zinvalid_valuesZ config_namevaluesr )rS rP r tagset_validatord s
r[ )r4
expected_typec C s d}t d }||kr>t| |s:d| d| dt | dS |S |tkrt| tsldt d| dt | dS t| trt| dkrdt d| dt | d S |S nTt|jd
d d jd d }t|}|d
kr(xL|D ]D}t| |}|dk r|S |dkrd| d }q|d| d 7 }qW n| d krNd| d| dt | dS |tkrt| dkrjdS d| dt | dt| dS t| tt t
f st| dkrd| dt | dt| dS |dkrzt| tsd| dt | dt| dS |tt kr|\}} d}
d} x2| j D ]&\} }
|
t| |7 }
| t|
| 7 } q*W |
dksh| dkrd|
d| S n|t| t
t fsd| dt | dt| dS |tt
kr|d } d} x | D ]}
| t|
| 7 } qW | dkrd| S |S d S )N z
Expected `z`. Found value: `z ` of type `z`.
z ` of type: `r z!` with length > 0. Found value: `z` with length: 0.
.r [r ()z
OR
z%` of length 0. Found value of type: `z`, with length: z.
z)` with length > 0. Found value of type: `r zTyping errors with keys:
z and values:
z Typing errors with values:
z
OR
()typer/ strrY r! r
validate_type EmptyListrM r1 r0 r rO r
)r4 r\ error_stringZNoneTypeZexpected_type_nameZexpected_type_argsZtype_argZtemp_error_stringZkey_typeZ
value_typeZkey_error_stringZvalue_error_stringkrV r r r re sh
"
re )
metadata_dictc C s dd t tD }i }xR| j D ]F\}}t| | |j|ttt tttt f f }|dkr |||< q W t |dkrt
d| d S )Nc S s i | ]}|j |jqS r )rc rQ )r# fieldr r r r% s z*validate_metadata_type..r] r z'The following typing errors are found: ) r DatasetMetadatarO re getr r
rd r rY r5 )ri Z field_typesZ
typing_errors
field_nameZ field_valueZfield_type_errorr r r validate_metadata_type s , rn c @ s e Zd ZdS )_nothingN)r> r? r@ r r r r ro s ro c @ sN e Zd ZU ee eeee f eeee f ee
ee e ee
ee ee eeee f dZee dZeee dZeee dZee dZee dZee dZeee dZeeeee f dZeeeee f dhZ e e e Z e e ddgZ dd Z!dd Z"e#e$d d d
d Z%e#ee&d d
dZ'e#ed d ddZ(e)eee eeee f f e*dddZ+e)eee eeee f f e*dddZ,e)eee eeee f f e*dddZ-e)eee eeee f f e*ddd Z.e)eee eeee f f e*d d d Z/e)eee eeee f f e*d d!d"Z0e)eee eeee f f e*d#d$d%Z1e)eee eeee f f e*d&d'd(Z2e)eee eeee f f e*d)d*d+Z3e)ee e*d,d-d.Z4e)eeeeef f d/d0d1Z5e)ee d2d3d4Z6dS )5rk Ntrain_eval_indexr' languagesc C s, | j d k rtjd | jd k r(tjd d S )NzFThe 'licenses' YAML field is deprecated, please use 'license' instead.zHThe 'languages' YAML field is deprecated, please use 'language' instead.)r' warningswarningrq )r- r r r
__post_init__ s
z DatasetMetadata.__post_init__c C s^ t t| d | j| j\| _}| j| j\| _}| j| jp@| j\| _}| j | j
pX| j \| _
}| j | j
\| _
}| j| j\| _}| j| j\| _}| j| j\| _}| j| j\| _} | j| j\| _}
| j| j\| _} |||||||| ||
| d } t }
x(| j D ] \}}|d k r ||
|< q W t |
dkrZt ddj dd |
j D d S )N)ri ) annotations_creatorslanguage_creatorslicensemultilingualitysize_categoriessource_datasetstask_categoriestask_idslanguagepaperswithcode_idrp r z=Could not validate the metadata, found the following errors:
rD c s s" | ]\}}d | d| V qdS )z * field 'z':
Nr )r# fieldnameerrr r r , s z+DatasetMetadata.validate..) rn vars validate_annotations_creatorsru validate_language_creatorsrv validate_language_codesr} rq validate_licencesrw r' validate_mulitlingualityrx validate_size_catgeoriesry validate_source_datasetsrz validate_task_categoriesr{ validate_task_idsr| !validate_paperswithcode_id_errorsr~ validate_train_eval_indexrp rM rO rY r5 rG )r- Zannotations_creators_errorsZlanguage_creators_errorsZlanguage_errorsZlicense_errorsZmultilinguality_errorsZsize_categories_errorsZsource_datasets_errorsZtask_categories_errorsZtask_ids_errorsZpaperswithcode_id_errorsZtrain_eval_index_errorserrorsZexception_msg_dictZ
yaml_fieldZerrsr r r validate sB
zDatasetMetadata.validate)rB r c C s. t |}|dk r| j|S td| ddS )a Loads and validates the dataset metadat from its dataset card (README.md)
Args:
path (:obj:`Path`): Path to the dataset card (its README.md file)
Returns:
:class:`DatasetMetadata`: The dataset's metadata
Raises:
:obj:`TypeError`: If the dataset card has no metadata (no YAML header)
:obj:`TypeError`: If the dataset's metadata is invalid
Nz Unable to find a yaml block in '')rI from_yaml_stringr5 )clsrB Z yaml_stringr r r from_readme/ s
zDatasetMetadata.from_readme)stringr c s t j|tdpt } fdd|D }|r@td| d j fdd|D }|rltd| d j fdd|j D }|S ) a+ Loads and validates the dataset metadat from a YAML string
Args:
string (:obj:`str`): The YAML string
Returns:
:class:`dict`: The dataset's metadata as a dictionary
Raises:
:obj:`TypeError`: If the dataset's metadata is invalid
)rJ c s g | ]}| j kr|qS r )_ALLOWED_YAML_KEYS)r# rh )r r r r. S s zCDatasetMetadata._metadata_dict_from_yaml_string..zBad YAML keys: z. Allowed fields: c s g | ]}| j kr|qS r )r )r# rh )r r r r. X s c s4 i | ],\}}||j d d jkr,|j d dn|qS )-r, )replace_FIELDS_WITH_DASHES)r# r2 r4 )r r r r% ] s zCDatasetMetadata._metadata_dict_from_yaml_string..)rK rL r+ rM
ValueErrorr rO )r r ri Zbad_keysr )r r _metadata_dict_from_yaml_stringC s
z/DatasetMetadata._metadata_dict_from_yaml_stringc C s | j |}| f |S )ak Loads and validates the dataset metadata from a YAML string
Args:
string (:obj:`str`): The YAML string
Returns:
:class:`DatasetMetadata`: The dataset's metadata
Raises:
:obj:`TypeError`: If the dataset's metadata is invalid
:obj:`ValueError`: If the dataset's metadata is invalid
)r )r r ri r r r r c s
z DatasetMetadata.from_yaml_string)ru r c C s t | td dtS )N annotationsru )r[ known_creatorsknown_creators_url)ru r r r r t s z-DatasetMetadata.validate_annotations_creators)rv r c C s t | td dtS )Nr} rv )r[ r r )rv r r r r z s z*DatasetMetadata.validate_language_creators)rq r c C s t | tj dtdd S )Nr} c S s | dkS )Nunknownr )langr r r s z9DatasetMetadata.validate_language_codes..)r[ known_language_codesr7 known_language_codes_url)rq r r r r ~ s z'DatasetMetadata.validate_language_codes)r' r c C s" t | ttj dt\}}||fS )Nrw )r[ r0 known_licensesr7 known_licenses_url)r' validatederrorr r r r s
z!DatasetMetadata.validate_licences)r{ r c C s, t tj }t| |dtdd \}}||fS )Nr{ c S s
| j dS )Nzother-)
startswith)er r r r s z:DatasetMetadata.validate_task_categories..)r0 known_task_idsr7 r[ known_task_ids_url)r{ known_setr r r r r r s z(DatasetMetadata.validate_task_categories)r| r c C s2 dd t j D }t| |dtdd \}}||fS )Nc S s& g | ] \}}|j d g D ]}|qqS )Zsubtasks)rl )r# Z_catdtidr r r r. s z5DatasetMetadata.validate_task_ids..r| c S s | pd| kp| j dS )Nz-other-zother-)r )r r r r r s z3DatasetMetadata.validate_task_ids..)r rO r[ r )r| r r r r r r r s z!DatasetMetadata.validate_task_ids)rx r c C s( t | ttj dtdd \}}||fS )Nrx c S s
| j dS )Nzother-)r )r r r r r s z:DatasetMetadata.validate_mulitlinguality..)r[ r0 known_multilingualitiesr7 known_size_categories_url)rx r r r r r r s
z(DatasetMetadata.validate_mulitlinguality) size_catsr c C s t | tdtS )Nry )r[ known_size_categoriesr )r r r r r s z(DatasetMetadata.validate_size_catgeories)sourcesr c C s t | tdtS )Nrz )r[ known_source_datasetsknown_source_datasets_url)r r r r r s z(DatasetMetadata.validate_source_datasets)r~ r c C s@ | d kr| d fS d| ks$| j | kr4d d| dfS | d fS d S )N zHThe paperswithcode_id must be lower case and not contain spaces but got zZ. You can find the paperswithcode_id in the URL of the dataset page on paperswithcode.com.)lower)r~ r r r r s z1DatasetMetadata.validate_paperswithcode_id_errors) pretty_namec C sn t | tr t| dkrjdS nNd}x0| j D ]$\}}t|dkr*|d| d7 }q*W |dkrbd |fS | d fS d S )Nr JThe pretty name must have a length greater than 0 but got an empty string.r] zVThe pretty name must have a length greater than 0 but got an empty string for config: z.
)Nr )r/ rd rY rO )r rg r2 r4 r r r validate_pretty_name s
z$DatasetMetadata.validate_pretty_name)rp c C s4 | d k r(t | t r(d dt| dfS | d fS d S )Nz)train-eval-index must be a list, but got z instead.)r/ r0 rc )rp r r r r s z)DatasetMetadata.validate_train_eval_index)7r> r? r@ r
rd ru r rf rv r} rw rx r ry rz r{ r| r~ r rp r ZconfigsZextra_gated_fieldsZextra_gated_promptZlicense_detailstagsr' rq r r setr Z_DEPRECATED_YAML_KEYSrt r classmethodr r rM r r staticmethodValidatorOutputr r r r r r r r r r r r r r r r rk sl
+ *** * * *
*
** rk c C s, h | ]$}|j tjkr"|j jd dn|j qS )r, r )rQ rk r r )r# rj r r r s r __main__)ArgumentParserz5Validate the yaml metadata block of a README.md file.)usagereadme_filepath)N)Lr loggingrW rr collectionsr Z dataclassesr r pathlibr typingr r r r r
r r r
r r ImportErrorZimportlib.resourcesr r Zimportlib_resourcesrK r] r __file__Zthis_url getLoggerr> loggerrd r r( r r r r r r r r r r r Zknown_multilingualities_urlr r Z
SafeLoaderr+ rI rN r boolr[ re rM rn ro rf rk r argparser Zap add_argument
parse_argsargsr r Zdataset_metadatar r r r r sh ,
6J