/
usr
/
local
/
lib64
/
python3.6
/
site-packages
/
tokenizers
/
normalizers
/
/usr/local/lib64/python3.6/site-packages/tokenizers/normalizers
mkdir
upload
Name
Size
Mode
Actions
__pycache__/
-
0755
rm
__init__.py
807
0644
edit
dl
rm
__init__.pyi
18295
0644
edit
dl
rm
Edit:
/usr/local/lib64/python3.6/site-packages/tokenizers/normalizers/__init__.py
(807B)
from .. import normalizers Normalizer = normalizers.Normalizer BertNormalizer = normalizers.BertNormalizer NFD = normalizers.NFD NFKD = normalizers.NFKD NFC = normalizers.NFC NFKC = normalizers.NFKC Sequence = normalizers.Sequence Lowercase = normalizers.Lowercase Strip = normalizers.Strip StripAccents = normalizers.StripAccents Nmt = normalizers.Nmt Precompiled = normalizers.Precompiled Replace = normalizers.Replace NORMALIZERS = {"nfc": NFC, "nfd": NFD, "nfkc": NFKC, "nfkd": NFKD} def unicode_normalizer_from_str(normalizer: str) -> Normalizer: if normalizer not in NORMALIZERS: raise ValueError( "{} is not a known unicode normalizer. Available are {}".format( normalizer, NORMALIZERS.keys() ) ) return NORMALIZERS[normalizer]()
Save
cmd:
run