FM / __pycache_

/usr/local/lib/python3.6/site-packages/sacremoses/__pycache__/

/usr/local/lib/python3.6/site-packages/sacremoses/__pycache__

Name	Size	Mode	Actions
chinese.cpython-36.pyc	14710	0644	edit dl rm
cli.cpython-36.pyc	6519	0644	edit dl rm
corpus.cpython-36.pyc	4915	0644	edit dl rm
indic.cpython-36.pyc	949	0644	edit dl rm
normalize.cpython-36.pyc	4829	0644	edit dl rm
sent_tokenize.cpython-36.pyc	721	0644	edit dl rm
subwords.cpython-36.pyc	4650	0644	edit dl rm
tokenize.cpython-36.pyc	16593	0644	edit dl rm
truecase.cpython-36.pyc	11778	0644	edit dl rm
util.cpython-36.pyc	5130	0644	edit dl rm
__init__.cpython-36.pyc	299	0644	edit dl rm
__main__.cpython-36.pyc	224	0644	edit dl rm

Edit: /usr/local/lib/python3.6/site-packages/sacremoses/__pycache__/tokenize.cpython-36.pyc (16593B)

3 �Eg�z�@s�ddlZddlmZddlmZddlmZddlmZddlm Z m Z e�Ze�ZGdd�de �ZGd d �d e �Zdd gZdS)�N)� text_type)�Perluniprops)�NonbreakingPrefixes)�is_cjk)�VIRAMAS�NUKTASc2seZdZdZedjejd��Zedjejd��dje �dje ��Zedjejd��Zedjejd��Z edjejd��dje �dje ��Zedjejd��Zd�Zd�Zd�Zd�Zd�Zdje�dfZdjed�dfZd�Zd�Zd�Zdje�dfZdje�dfZdje�dfZd�Zd�Zd�Z d�Z!d�Z"d�Z#d�Z$d�Z%d�Z&d�Z'd0jed1�d2fZ(d3jed1�d2fZ)d4jed1�d2fZ*d5jee �dfZ+d6jed�d7fZ,d�Z-d�Z.d�Z/d�Z0d�Z1d�Z2d�Z3d�Z4d�Z5d�Z6d�Z7d�Z8d�Z9d�Z:d�Z;d�Zd�Z?d�Z@d�ZAd�ZBd�ZCd�ZDd�ZEd�ZFd�ZGd�ZHd�ZId�ZJd�ZKd�ZLd�ZMd�ZNd�ZOd�ZPd�ZQd�ZRd�ZSd�ZTd�ZUd�ZVd�ZWd�ZXd�ZYd�jed��d�fZZd�jeed��d�fZ[d�jed��d�fZ\d�jed��d�fZ]d�jed��d�fZ^eZe[e\e]e^gZ_d�jed��d�fZ`d�jed��d�fZad�jed��d�fZbd�jed��d�fZce`eaebecgZdd�Zed�Zfd�Zgd�Zhd�Zid�Zjd�Zkeeeee e!e"e#e$e%e&e(e)e*e+e,e-e.e/e0e1e2e3e4e5e6e7e8e9e:e;ee?e@eAeBeCeDeEeFeGeHeIeJeKeLeMeNg2Zle'eOePeQeReSeTeUeVeWg ZmeReSeTeUeVeWeXeYgZnegeheiejekgZod�d�d�d�gZpd��fd�d�� Zqd�d��Zrd�d��Zsd�d��Ztd�d��Zud�d��Zvd�d��Zwd�d��Zxd�d�d��Zyd�d�d��Zz�Z{S)��MosesTokenizerz� This is a Python port of the Moses Tokenizer from https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl ��IsN�IsAlnum�IsSc�IsSo�IsAlpha�IsLower�\s+� �[\000-\037]� +�^ � $z([^{}\s\.'\`\,\-])� \1 z ([{alphanum}])\-(?=[{alphanum}]))�alphanumz\1 @-@ � \.([\.]+)� DOTMULTI�DOTMULTI\.([^\.])� DOTDOTMULTI � DOTMULTI\.�DOTDOTMULTIz ([^{}])[,]z\1 , z [,]([^{}])z , \1z ([{}])[,]$�^``�`` �^"�^`([^`])�` \1�^'�` � ([ ([{<])"�\1 `` �([ ([{<])``�([ ([{<])`([^`])�\1 ` \2� ([ ([{<])'�\1 ` �\.\.\.� _ELLIPSIS_ � _ELLIPSIS_z([^{numbers}])[,]([^{numbers}]))Znumbersz\1 , \2z([{numbers}])[,]([^{numbers}])z([^{numbers}])[,]([{numbers}])z([;:@#\$%&{}{}])z([{alphanum}])\/([{alphanum}])z$1 \@\/\@ $2�([^.])([.])([\]\)}>"']*) ?$�\1 \2\3�([?!])�([\]\[{}<>])�$�-LRB-�$�-RRB-�\[�-LSB-�\]�-RSB-�\{�-LCB-�\}�-RCB-�--� -- �^�$�"� '' �([^'])' �\1 ' �([^'])'�'([sSmMdD]) � '\1 �'ll � 'll �'re � 're �'ve � 've �n't � n't �'LL � 'LL �'RE � 'RE �'VE � 'VE �N'T � N'T � ([Cc])annot � \1an not � ([Dd])'ye � \1' ye � ([Gg])imme � \1im me � ([Gg])onna � \1on na � ([Gg])otta � \1ot ta � ([Ll])emme � \1em me � ([Mm])ore'n � \1ore 'n � '([Tt])is � '\1 is � '([Tt])was � '\1 was � ([Ww])anna � \1an na � *�^ *� *$�&�&�\|�|�<�<�>�>�\'�'�\"�"�[�]�]z([^{alpha}])[']([^{alpha}]))�alphaz\1 ' \2z([^{alpha}{isn}])[']([{alpha}]))r��isnz([{alpha}])[']([^{alpha}])z([{alpha}])[']([{alpha}])z\1 '\2z([{isn}])[']([s]))r�z([^{alpha}])[']([{alpha}])z\1' \2� ' �\.' ?$� . ' z<\/?\S+\/?>z#<\S+( [a-zA-Z0-9]+\="?[^"]")+ ?\/?>z#<\S+( [a-zA-Z0-9]+\='?[^']')+ ?\/?>z'[\w\-\_\.]+\@([\w\-\_]+\.)+[a-zA-Z]{2,}z/(http[s]?|ftp):\/\/[^:\/\s]+(\/\w+)*\/[\w\-\.]+z"((https?|ftp|rsync)://|www\.)[^ ]*z@[a-zA-Z0-9_]+z#[a-zA-Z0-9_]+�enNcs�tt��j�|�_dd�tj|�D��_|r�g�_t|d��B}x:|D]2}|j�}|rF|j d�rF|�jkrF�jj |�qFWWdQRX�fdd��jD��_�jdk�r�d }�jdkr�|td j tjd��7}�jdk�r�|td j tjd��7}�jdk�rF|td j tjd ��7}|td j tjd��7}|td j tjd��7}�j|7_�j|7_dj�j�df�_dj�jd�df�_dj�jd�df�_dS)NcSsg|]}|j��qS�)�strip)�.0Z_nbpr�r��=/usr/local/lib/python3.6/site-packages/sacremoses/tokenize.py� 1sz+MosesTokenizer.__init__..�r�#cs$g|]}�j|�r|jd�d�qS)rr)�has_numeric_only� rpartition)r��w)�selfr�r�r�?s�zh�ja�ko�cjkr ZHangul�Han�Hiragana�Katakanaz([^{}\s\.'\`\,\-])z \1 z ([{alphanum}])\-(?=[{alphanum}]))rz\1 @-@ z([{alphanum}])\/([{alphanum}])z$1 \@\/\@ $2)r�r�r�r�)r�r�)r�r�)r�r�)�superr�__init__�lang�nonbreaking_prefixes�words�NONBREAKING_PREFIXES�openr�� startswith�append�NUMERIC_ONLY_PREFIXESr�join�perluniprops�charsrr�format�PAD_NOT_ISALNUM�AGGRESSIVE_HYPHEN_SPLIT�INTRATOKEN_SLASHES)r�r�Z custom_nonbreaking_prefixes_fileZfin�lineZ cjk_chars)� __class__)r�r�r�*s< zMosesTokenizer.__init__cCs@tjdd|�}x,tjd|�r:tjdd|�}tjdd|�}qW|S)Nz \.([\.]+)z DOTMULTI\1z DOTMULTI\.zDOTMULTI\.([^\.])zDOTDOTMULTI \1r)�re�sub�search)r��textr�r�r��replace_multidots[s z MosesTokenizer.replace_multidotscCs.xtjd|�rtjdd|�}qWtjdd|�S)Nrz DOTMULTI.ZDOTMULTI�.)r�r�r�)r�r�r�r�r��restore_multidotsbsz MosesTokenizer.restore_multidotscCst|�jt|j��S)N)�set� differencer)r�r�r�r�r��islowergszMosesTokenizer.islowercCstt|�jt|j��S)N)�anyr��intersectionr)r�r�r�r�r�� isanyalphajszMosesTokenizer.isanyalphacCsttjd|��S)Nz[\s]+(\#NUMERIC_ONLY\#))�boolr�r�)r�r�r�r�r�r�mszMosesTokenizer.has_numeric_onlycCs�|j�}t|�}x�t|�D]�\}}tjd|�}|r|jd�}d|krN|j|�s�||jkrb||jks�||dkr�||dr�|j ||dd�r�q||jkr�|d|kr�tjd||d�r�q|d||<qWdj |�S)Nz ^(\S+)\.$�r�rz^[0-9]+z .r)�split�len� enumerater�r��groupr�r�r�r�r�)r�r��tokensZ num_tokens�i�tokenZtoken_ends_with_period�prefixr�r�r��handles_nonbreaking_prefixesps& z+MosesTokenizer.handles_nonbreaking_prefixescCs&x |jD]\}}tj|||�}qW|S)N)�MOSES_ESCAPE_XML_REGEXESr�r�)r�r��regexp�substitutionr�r�r�� escape_xml�szMosesTokenizer.escape_xmlFcCsft|�}x |jD]\}}tj|||�}qW|j|�}x |jD]\}}tj|||�}q.ZTHISISPROTECTED�r��fr�it)r�r�)r�DEDUPLICATE_SPACE� ASCII_JUNKr�r�r��str�zfill�replacer�r�r�r��COMMA_SEPARATE_1�COMMA_SEPARATE_2�COMMA_SEPARATE_3r��ENGLISH_SPECIFIC_APOSTROPHE�FR_IT_SPECIFIC_APOSTROPHE�NON_SPECIFIC_APOSTROPHEr��TRAILING_DOT_APOSTROPHEr�r�r�)r�r�Zaggressive_dash_splitsr��escapeZprotected_patternsr�r�Zprotected_tokensr�r�Z substituitionr�)r�r��tokenize�sT zMosesTokenizer.tokenize)rr)rr )rr)rr )rr )rr)rr)rr)rr)r r)r!r")r#r$)r%r&)r'r&)r(r))r*r+)r,r-)r.r,)r/r0)r1r)r2r)r3r4)r5r6)r7r8)r9r:)r;r<)r=r>)r?r@)rAr)rBr)rCrD)rErF)rGrF)rHrI)rJrK)rLrM)rNrO)rPrQ)rRrS)rTrU)rVrW)rXrY)rZr[)r\r])r^r_)r`ra)rbrc)rdre)rfrg)rhri)rjrk)rlrm)rnr)ror )rpr )rqrr)rsrt)rurv)rwrx)ryrz)r{r|)r7r})r~r)ryr�)r�r�)r�N)F)FFTN)|�__name__� __module__�__qualname__�__doc__rr�r�r�r rrrrr rrr�r�Z MID_STRIPZ LEFT_STRIPZRIGHT_STRIPr�r�r�Z REPLACE_DOT_WITH_LITERALSTRING_1Z REPLACE_DOT_WITH_LITERALSTRING_2Z REPLACE_DOT_WITH_LITERALSTRING_3r�r�r�ZDIRECTIONAL_QUOTE_1ZDIRECTIONAL_QUOTE_2ZDIRECTIONAL_QUOTE_3ZDIRECTIONAL_QUOTE_4ZDIRECTIONAL_QUOTE_5ZDIRECTIONAL_QUOTE_6ZDIRECTIONAL_QUOTE_7ZDIRECTIONAL_QUOTE_8ZREPLACE_ELLIPSISZRESTORE_ELLIPSISZCOMMA_1ZCOMMA_2ZCOMMA_3ZSYMBOLSr�ZFINAL_PERIODZPAD_QUESTION_EXCLAMATION_MARKZPAD_PARENTHESISZCONVERT_PARENTHESIS_1ZCONVERT_PARENTHESIS_2ZCONVERT_PARENTHESIS_3ZCONVERT_PARENTHESIS_4ZCONVERT_PARENTHESIS_5ZCONVERT_PARENTHESIS_6ZPAD_DOUBLE_DASHESZPAD_START_OF_STRZPAD_END_OF_STRZCONVERT_DOUBLE_TO_SINGLE_QUOTESZHANDLES_SINGLE_QUOTESZ APOSTROPHEZ CONTRACTION_1Z CONTRACTION_2Z CONTRACTION_3Z CONTRACTION_4Z CONTRACTION_5Z CONTRACTION_6Z CONTRACTION_7Z CONTRACTION_8Z CONTRACTION_9ZCONTRACTION_10ZCONTRACTION_11ZCONTRACTION_12ZCONTRACTION_13ZCONTRACTION_14ZCONTRACTION_15ZCONTRACTION_16ZCONTRACTION_17ZCONTRACTION_18ZCONTRACTION_19ZCLEAN_EXTRA_SPACE_1ZCLEAN_EXTRA_SPACE_2ZCLEAN_EXTRA_SPACE_3ZESCAPE_AMPERSANDZESCAPE_PIPEZESCAPE_LEFT_ANGLE_BRACKETZESCAPE_RIGHT_ANGLE_BRACKETZESCAPE_SINGLE_QUOTEZESCAPE_DOUBLE_QUOTEZESCAPE_LEFT_SQUARE_BRACKETZESCAPE_RIGHT_SQUARE_BRACKETZ EN_SPECIFIC_1Z EN_SPECIFIC_2Z EN_SPECIFIC_3Z EN_SPECIFIC_4Z EN_SPECIFIC_5r�ZFR_IT_SPECIFIC_1ZFR_IT_SPECIFIC_2ZFR_IT_SPECIFIC_3ZFR_IT_SPECIFIC_4r�r�r�ZBASIC_PROTECTED_PATTERN_1ZBASIC_PROTECTED_PATTERN_2ZBASIC_PROTECTED_PATTERN_3ZBASIC_PROTECTED_PATTERN_4ZBASIC_PROTECTED_PATTERN_5r�r�r�ZBASIC_PROTECTED_PATTERNSZWEB_PROTECTED_PATTERNSr�r�r�r�r�r�r�r�r�r�� __classcell__r�r�)r�r�rs�&& 1) rc6s�eZdZdZedjejd��Zedjejd��Z edjejd��Z djZej d�d fZdkZdlZdmZdnZdoZdpZdqZdrZdsZdtZduZeeeeeeeeeeegZdddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;dd?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRg6ZdSdTdUdVdWgZdXdYd/d1dZd[d\d]d^g Zd_jedje��edje��edje��Zdv�fdadb� Z dcdd�Z!dwdfdg�Z"dxdhdi�Z#�Z$S)y�MosesDetokenizerz� This is a Python port of the Moses Detokenizer from https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/detokenizer.perl r rrr� \@\-\@ �-z {2,}r�|�|�<ru�>rw�"rC�'�'�[�[�]r~�&rq�&bar;�&bra;�&ket;�N�n�A�a�Ä�äZssaZSsaussäuSsäZstaustäZStauStäZhunZHunZhynZHynZhanr�uhänuHänuhönuHön�unZUnZynZYnZanZAnuänuÄnuönuÖn�seenZSeenZllaZLlaulläuLläZltaZLtaultäuLtäZlleZLleZksiZKsiZkseZKseZttaZTtaZineZIne�ni�siZmmeZnneZnsar�ukö�paupäZkaanukäänZkinz^({})({})?({})$r�cstt|�j�||_dS)N)r�r�r�r�)r�r�)r�r�r�r��szMosesDetokenizer.__init__cCs&x |jD]\}}tj|||�}qW|S)N)�MOSES_UNESCAPE_XML_REGEXESr�r�)r�r�r�r�r�r�r��unescape_xml�szMosesDetokenizer.unescape_xmlTc Cs<djdj|��}t|�}|j\}}tj|||�}|r>|j|�}dddddd�}d}d} |j�}�x�tt |��D�]�\} }t |d�r�|jdkr�| dkr�t || dd$�r�| |7} n| ||7} d}qntjd|j d |�r�| ||7} d}qntjd |��r,|jdk�rtjd|��r| d7} | |7} d}qn|jd k�rf| dk�rftjdj|j�|��rf| |7} d}qn|jdk�r�| dk�r�tjd|d%��r�tjd|d&��r�tjd|��r�| |7} d}qn|jd'k�r | t|�dk�r tjdj|j�|��r tjdj|j�|| d��r | ||7} d}qn|jdk�r�| t|�dk�r�tjdj|j�|��r�tjd|| d��r�tjd|| dtj��r�| |||| d7} t|d�d}qntjd|��r�|}tjd|��r�d}|j|d�||<|jdk�r�|dk�r�d||<|jdk�r|dk�rd||<||ddk�r�|jd k�rr|d k�rr| dk�rrtjd!|| d��rr| |7} d}n | ||7} d}||d7<n| |7} d}||d7<qn|jd"k�r�tjd#|| d��r�tj|j|��r�| ||7} d}qn| ||7} d}qnW|j\}}tj||| �} | j�} |�r4| S| j�S)(z� Python port of the Moses detokenizer. :param tokens: A list of strings, i.e. tokenized text. :type tokens: list(str) :return: str z {} rr)r�rCz``�`z''r r�r�z^[u$\[\{\¿\¡]+$z^[\,\.\?\!\:\;\\\%\}\]$]+$r�z^[\?\!\:\;\\\%]$r�z^['][{}]�csz^[0-9]+$�z^[.,]$r��gaz[{}][']$z^[{}]r�u^[-–]$z^li$|^mail.*Nu^[\'\"„“`]+$u^[„“”]+$rCu„u“r�z[s]$�fiz:$��r)r�r�r)r�r�rr�r�r�rr�r��iterrr�r�rrr�r��next�get� FINNISH_REGEX� ONE_SPACEr�) r�r�r��unescaper�r�r�Zquote_countsZ prepend_spaceZdetokenized_textr�r�Znormalized_quor�r�r�r��s� zMosesDetokenizer.tokenizecCs|j|||�S)z&Duck-typing the abstract *tokenize()*.)r�)r�r�r�rr�r�r�� detokenize8szMosesDetokenizer.detokenize)r�r�)r�r�)r�ru)r�rw)r�rC)r�r�)r�r�)r�r~)r�rq)r�r�)r�r�)r�r~)r�)TT)TT)%r�r�r�r�rr�r�r�rrrr�r��compiler ZUNESCAPE_FACTOR_SEPARATORZUNESCAPE_LEFT_ANGLE_BRACKETZUNESCAPE_RIGHT_ANGLE_BRACKETZUNESCAPE_DOUBLE_QUOTEZUNESCAPE_SINGLE_QUOTEZ UNESCAPE_SYNTAX_NONTERMINAL_LEFTZ!UNESCAPE_SYNTAX_NONTERMINAL_RIGHTZUNESCAPE_AMPERSANDZ UNESCAPE_FACTOR_SEPARATOR_LEGACYZ'UNESCAPE_SYNTAX_NONTERMINAL_LEFT_LEGACYZ(UNESCAPE_SYNTAX_NONTERMINAL_RIGHT_LEGACYrZFINNISH_MORPHSET_1ZFINNISH_MORPHSET_2ZFINNISH_MORPHSET_3r�rr�rr�rr�r�r�)r�r�r�s� r�)r��sixrZsacremoses.corpusrrZsacremoses.utilrZsacremoses.indicrrr�r��objectrr��__all__r�r�r�r��s #