FM / roformer

/usr/local/lib/python3.6/site-packages/transformers/models/roformer/

/usr/local/lib/python3.6/site-packages/transformers/models/roformer

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
configuration_roformer.py	6832	0644	edit dl rm
convert_roformer_original_tf_checkpoint_to_pytorch.py	2212	0644	edit dl rm
modeling_flax_roformer.py	39026	0644	edit dl rm
modeling_roformer.py	67178	0644	edit dl rm
modeling_tf_roformer.py	61321	0644	edit dl rm
tokenization_roformer.py	14257	0644	edit dl rm
tokenization_roformer_fast.py	8145	0644	edit dl rm
tokenization_utils.py	2652	0644	edit dl rm
__init__.py	4409	0644	edit dl rm

Edit: /usr/local/lib/python3.6/site-packages/transformers/models/roformer/tokenization_utils.py (2652B)

# coding=utf-8
# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenization utils for RoFormer."""

from typing import List

from tokenizers import NormalizedString, PreTokenizedString, normalizers

class JiebaPreTokenizer:
    def __init__(self, vocab) -> None:
        self.vocab = vocab
        self.normalizers = normalizers.BertNormalizer(
            clean_text=False,
            handle_chinese_chars=True,
            strip_accents=False,
            lowercase=False,
        )
        try:
            import rjieba
        except ImportError:
            raise ImportError(
                "You need to install rjieba to use RoFormerTokenizer. "
                "See https://pypi.org/project/rjieba/ for installation."
            )
        self.jieba = rjieba

def jieba_split(self, i: int, normalized_string: NormalizedString) -> List[NormalizedString]:
        splits = []

# this code slice normalized_string is too slow (6s) but test_alignement_methods can pass
        for token, start, end in self.jieba.tokenize(str(normalized_string), hmm=False):
            if token in self.vocab:
                splits.append(normalized_string[start:end])
            else:
                token_list = self.normalizers.normalize_str(token).split()
                for token in token_list:
                    if token:
                        end = start + len(token)
                        splits.append(normalized_string[start:end])
                        start = end

# this code test_alignement_methods can't pass but fast (300ms)
        # for token in self.jieba.cut(str(normalized_string), False):
        #     if token in self.vocab:
        #         splits.append(NormalizedString(token))
        #     else:
        #         token_list = self.normalizers.normalize_str(token).split()
        #         for token in token_list:
        #             if token:
        #                 splits.append(NormalizedString(token))

return splits

def pre_tokenize(self, pretok: PreTokenizedString):
        pretok.split(self.jieba_split)

cmd: