Name	Size	Mode	Actions
chinese.cpython-36.pyc	14710	0644	editdlrm
cli.cpython-36.pyc	6519	0644	editdlrm
corpus.cpython-36.pyc	4915	0644	editdlrm
indic.cpython-36.pyc	949	0644	editdlrm
normalize.cpython-36.pyc	4829	0644	editdlrm
sent_tokenize.cpython-36.pyc	721	0644	editdlrm
subwords.cpython-36.pyc	4650	0644	editdlrm
tokenize.cpython-36.pyc	16593	0644	editdlrm
truecase.cpython-36.pyc	11778	0644	editdlrm
util.cpython-36.pyc	5130	0644	editdlrm
__init__.cpython-36.pyc	299	0644	editdlrm
__main__.cpython-36.pyc	224	0644	editdlrm

3 �Eg� �@sPdd
lm
Z

ddlZddlmZmZ
ddlmZ
ddlm Z
Gdd�de �ZdS)�)
�print_functionN)�Counter�defaultdict)
�reduce)
�pairwisec@sLeZ
dZd
Zdd�Zddd�
Zdd�Zd d �Zdd�Zd d�Z ddd�
Z dS)�SubwordTokenizerzd This is a Python port of the Subword NMT from https://github.com/rsennrich/subword-nmt cCs.|j|
�
|_
|j�\|_|_tj|j�
|_dS)
N)�get_vocabulary�vocab�get_pair_statistics�stats�indices�copy�deepcopyZ big_stats)�self�filename�r�=/usr/local/lib/python3.6/site-packages/sacremoses/subwords.py�__init__s

zSubwordTokenizer.__init__Fc Cs�t�}t
|
�
�R}|rJxF|D]*}|j�jd
�
\}}||t|�
7<qWn|j|j�j��

WdQRXtdd�|j�D�
�
}|j�S)N�
c
Ss0i|](\}
}|t|
dd��
|
ddf
�qS)N�
u�����r)
�tuple)�.0�
k�
vrrr� $sz3SubwordTokenizer.get_vocabulary..) r�open�strip�split�int�update�read�items�most_common)rr�is_dictr Zfin�line�word�countrrrrs






zSubwordTokenizer.get_vocabularyc
Csxt�}
t
d
d��
}x\t|j�
D]N\}\}}x@t|�
D]4\}}|
||f|7<|||f|d7<q4WqW|
|fS)z5Count frequency of all symbol pairs, and create indexc
Sst�S)
N)
rrrrr�-sz6SubwordTokenizer.get_pair_statistics..r)rr� enumerater r)rrr�
ir&�freq�prev�currrrrr (s


z$SubwordTokenizer.get_pair_statisticscsD|\��d
j|�
j
dd��
��
�fdd�}t||
dd�|
df
�S) z� From https://stackoverflow.com/a/40367074/610569 >>> modify_token(('s', 'h', 'e', 'r', 'l', 'o', 'c', 'k'), ('h', 'e')) ('S', 'he', 'r', 'l', 'o', 'c', 'k') ��
\z\\cs0|d�kr&|
�kr&|dd��
f
S||
f
S)Nrrrr)�acc�
e)�first�pair_str�secondrrr(?s
&
z/SubwordTokenizer.modify_token..rNr
)�join�replacer)r�token�pair�
fr)r2r3r4r�modify_token6s
zSubwordTokenizer.modify_tokencCshg}x^|j|
j
�D]L\}}|d
kr&q|j|\}}|j||
�}||f|j|<|j||||f�

qW|S)zJReplace all occurrences of a symbol pair ('A', 'B') with a new symbol 'AB'r)rr"r r:�append)rr8�changes�
jr+r&Znew_wordrrr�replace_pairEs







zSubwordTokenizer.replace_pairc CsLd
|j|
<t
�|j|
<|
\}}||}�x|D�]\}}}} d
} �
x$y|j|| �} Wntk rj


PYnX| t|�
dko�|| d|k�
r\| r�|| d| d�}|j|| 8<|j||d8<| t|�
dk�
rR|| d|k�
s| t|�
dk�
s|| d|k�
rR|| d| d�}|j|| 8<|j||d8<| d7} qD| d7} qDWd
} x�y|j|| �} Wntk �
r�


PYnX| �
r�|| d| d�}|j|| 7<|j||d7<| t|�
dk�r6|| d|k�r6|| | d�}|j|| 7<|j||d7<| d7} �
qnWq.WdS)z� Minimally update the indices and frequency of symbol pairs if we merge a pair of symbols, only pairs that overlap with occurrences of this pair are affected, and need to be updated. r
r��N)rrr�index� ValueError�len) rr8�changedr2r4Znew_pairr=r&Zold_wordr+r*r,Znexrrr�update_pair_statisticsQsP






"

 



 




$
z'SubwordTokenizer.update_pair_statisticsr?rNcCsxt|j
j��
d
}x`t|
�
D]T}|j
j|�
}xB|D]:\}} |j|�
} |j|| �
d|j
|<|j
||kr2dSq2WqWdS)N� r
)�maxr�values�ranger#r>rE)rZnum_symbolsZmin_freqZjumpr$� thresholdr*Zmost_freq_tokensr7r'r<rrr�learn�s







zSubwordTokenizer.learn)
F)r?rN)�__name__� __module__�__qualname__�__doc__rrr r:r>rErKrrrrrs [r)� __future__rr �collectionsrr� functoolsrZsacremoses.utilr�objectrrrrr�s