/usr/local/lib/python3.6/site-packages/sacremoses/__pycache__
Edit: /usr/local/lib/python3.6/site-packages/sacremoses/__pycache__/cli.cpython-36.pyc (6519B)
3
Eg# @ s d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZm Z d dl
m Z m Z d dl
mZ d dlmZ d dlZd dlZejd d k rd dlZejZejed
ed d gd
Zejdedejdddddejdddddejdddd dejd d dd d d!ej d"d# Z e ej j d$d d%k r
d2ddd?d0e$d@dA Z(e j&dBejdCd dddDd0ejdEdFdddGd0ejdHd5d ddId0ejdJd9d ddKd0e$dLdM Z)e j&dNejdOdPddQdRejdSd.d ddTd0ejdUd5d ddVd0e$dWdX Z*e j&dYejdOdPddZdRejdSd.d ddTd0ejdUd5d dd[d0e$d\d] Z+e j&d^ejd_d.d dd`d0e$dadb Z,dS )c N)deepcopy)partial)update_wrapper)MosesTokenizerMosesDetokenizer)MosesTruecaserMosesDetruecaser)MosesPunctNormalizer)parallelize_preprocess zTYou should really be using Python3!!! Tick tock, tick tock, https://pythonclock.org/z-hz--help)Zhelp_option_namesT)chainZcontext_settingsz
--languagez-lenz+Use language specific rules when tokenizing)defaulthelpz --processesz-j zNo. of processes.z
--encodingz-eutf8zSpecify encoding of file.z--quietz-qFzDisable progress bar.)is_flagr r c C s d S )N )languageencoding processesquietr r 8/usr/local/lib/python3.6/site-packages/sacremoses/cli.pycli# s r . c K s\ t jd|dD}|}x | D ]}|t|f|}qW |rNx|D ]}t j| q.new_func..processor)r )r r' )r& r r new_funcE s zprocessor..new_func)r )r& r r( r )r& r r' @ s r' c c sH |dkr"x:| D ]}||V qW n"x t || || dD ]
}|V q6W d S )Nr ) progress_bar)r
)r! funcr r lineoutliner r r parallel_or_notN s
r- tokenizez--aggressive-dash-splitsz-azTriggers dash split rules.)r r r z --xml-escapez-xz"Escape special characters for XML.z--protected-patternsz-pzXSpecify file with patters to be protected in tokenisation. Special values: :basic: :web:)r z--custom-nb-prefixesz-czjSpecify a custom non-breaking prefixes file, add prefixes to the default ones from the specified language.c C s| t ||d}|rZ|dkr |j}n:|dkr0|j}n*t|dd} dd | j D }W d Q R X t|jd|||d }
t| |
||S )
N)langZ custom_nonbreaking_prefixes_filez:basic:z:web:r )r c S s g | ] }|j qS r )strip).0patternr r r
s z!tokenize_file..T)
return_straggressive_dash_splitsescapeprotected_patterns)r ZBASIC_PROTECTED_PATTERNSZWEB_PROTECTED_PATTERNSopen readlinesr r. r- ) r! r r r Z
xml_escaper5 r7 Zcustom_nb_prefixesmosesr Zmoses_tokenizer r r
tokenize_file^ s $
r;
detokenizez--xml-unescapez$Unescape special characters for XML.c C s4 t |d}t|jd|d}ttttj| |||S )N)r/ T)r4 unescape)r r r< r- r mapstrsplit)r! r r r Z xml_unescaper: Zmoses_detokenizer r r detokenize_file s
rA normalizez--normalize-quote-commasz Normalize quotations and commas.z--normalize-numbersz-dzNormalize number.z--replace-unicode-punctsz2Replace unicode punctuations BEFORE normalization.z--remove-control-charsz.Remove control characters AFTER normalization.c
C s* t |||||d}t|j} t| | ||S )N)Znorm_quote_commasZ norm_numbersZpre_replace_unicode_punctZpost_remove_control_chars)r r rB r- )
r! r r r Znormalize_quote_commasZnormalize_numbersZreplace_unicode_punctsZremove_control_charsr: Zmoses_normalizer r r normalize_file s $
rC ztrain-truecasez --modelfilez-mz Filename to save the modelfile.)requiredr z--is-asrz)A flag to indicate that model is for ASR.z--possibly-use-first-tokenz*Use the first token as part of truecasing.c C s, t |d}|j| ||| d}|j| d S )N)is_asr)possibly_use_first_tokenr r) )r train
save_model) r! r r r modelfilerE rF r: modelr r r train_truecaser s
rK truecasez$Filename to save/load the modelfile.z1Use the first token as part of truecase training.c C sd t jj|s s & '