/usr/local/lib/python3.6/site-packages/datasets/features/__pycache__
Edit: /usr/local/lib/python3.6/site-packages/datasets/features/__pycache__/translation.cpython-36.pyc (5169B)
3
<%Eg @ st d dl mZmZ d dlmZmZmZmZmZm Z m
Z
d dl Z erLddl
mZ eG dd dZeG dd d ZdS )
) dataclassfield)
TYPE_CHECKINGAnyClassVarDictListOptionalUnionN ) FeatureTypec @ sv e Zd ZU dZee dZee dZ e
e dZ e
e e
d dddZedd Zedeedf f d d
d ZdS ) Translationa `FeatureConnector` for translations with fixed languages per example.
Here for compatiblity with tfds.
Input: The Translate feature accepts a dictionary for each example mapping
string language codes to string translations.
Output: A dictionary mapping string language codes to translations as `Text`
features.
Example:
```python
>>> # At construction time:
>>> datasets.features.Translation(languages=['en', 'fr', 'de'])
>>> # During data generation:
>>> yield {
... 'en': 'the cat',
... 'fr': 'le chat',
... 'de': 'die katze'
... }
```
NdictF)defaultinitreprc C s t jdd t| jD S )Nc S s i | ]}t j |qS )pastring).0langr r G/usr/local/lib/python3.6/site-packages/datasets/features/translation.py
, s z(Translation.__call__..)r structsorted languages)selfr r r __call__+ s zTranslation.__call__r )returnc s$ ddl m fddt| jD S )z2Flatten the Translation feature into a dictionary.r )Valuec s i | ]} d |qS )r r )r k)r r r r 2 s z'Translation.flatten..)featuresr r r )r r )r r flatten. s zTranslation.flatten)__name__
__module__ __qualname____doc__r strr idr dtyper pa_typer r _typer r
r r" r r r r r
s
r
c @ s e Zd ZU dZdZee dZee dZ ee
dZ e e
dZ
e e
ed dddZe
dd Zdd Zd
d Zed ee
d f f d
ddZdS ) TranslationVariableLanguagesa` `FeatureConnector` for translations with variable languages per example.
Here for compatiblity with tfds.
Input: The TranslationVariableLanguages feature accepts a dictionary for each
example mapping string language codes to one or more string translations.
The languages present may vary from example to example.
Output:
language: variable-length 1D tf.Tensor of tf.string language codes, sorted
in ascending order.
translation: variable-length 1D tf.Tensor of tf.string plain text
translations, sorted to align with language codes.
Example:
```python
>>> # At construction time:
>>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de'])
>>> # During data generation:
>>> yield {
... 'en': 'the cat',
... 'fr': ['le chat', 'la chatte,']
... 'de': 'die katze'
... }
>>> # Tensor returned :
>>> {
... 'language': ['en', 'de', 'fr', 'fr'],
... 'translation': ['the cat', 'die katze', 'la chatte', 'le chat'],
... }
```
Nr F)r r r c C s< | j r tttt| j nd | _ | j r2t| j nd | _d S )N)r listr setlen
num_languages)r r r r
__post_init___ s "z*TranslationVariableLanguages.__post_init__c C s$ t jt jt j t jt j dS )N)language translation)r r list_r )r r r r r c s z%TranslationVariableLanguages.__call__c s t | j}| jrJt || rJtddjtt || ddj| dg }xF|j D ]:\ }t|trz|j |f qX|j fdd|D qXW t
t| \}}||dS )NzSome languages in example (z, z) are not in valid set (z).c s g | ] } |fqS r r )r el)r r r
t s z?TranslationVariableLanguages.encode_example..)r2 r3 ) r. r
ValueErrorjoinr items
isinstancer' appendextendzip)r Ztranslation_dictZlang_setZtranslation_tuplestextr Z translationsr )r r encode_examplef s
,
z+TranslationVariableLanguages.encode_exampler )r c C s* ddl m}m} ||d||ddS )zCFlatten the TranslationVariableLanguages feature into a dictionary.r )Sequencer r )r2 r3 )r! r@ r )r r@ r r r r r" { s
z$TranslationVariableLanguages.flatten)r# r$ r% r& r r r r0 intr( r' r) r r* r r r+ r1 r r? r
r r" r r r r r, 5 s
r, )Z dataclassesr r typingr r r r r r r
Zpyarrowr r! r r
r, r r r r s $ *