


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__lowerCAmelCase		       =							logging.get_logger(__name__)

__lowerCAmelCase		       =							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__lowerCAmelCase		       =							{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

__lowerCAmelCase		       =							{
    "junnyu/roformer_chinese_small": 15_36,
    "junnyu/roformer_chinese_base": 15_36,
    "junnyu/roformer_chinese_char_small": 5_12,
    "junnyu/roformer_chinese_char_base": 5_12,
    "junnyu/roformer_small_discriminator": 1_28,
    "junnyu/roformer_small_generator": 1_28,
}


__lowerCAmelCase		       =							{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}





class   __a			(    lowerCAmelCase_      ):
 __lowercase :							List[Any]        = VOCAB_FILES_NAMES
 __lowercase :							List[Any]        = PRETRAINED_VOCAB_FILES_MAP
 __lowercase :							Optional[int]        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase :							List[str]        = PRETRAINED_INIT_CONFIGURATION
 __lowercase :							str        = RoFormerTokenizer
 def __init__(   self				, lowerCAmelCase__=None				, lowerCAmelCase__=None				, lowerCAmelCase__=True				, lowerCAmelCase__="[UNK]"				, lowerCAmelCase__="[SEP]"				, lowerCAmelCase__="[PAD]"				, lowerCAmelCase__="[CLS]"				, lowerCAmelCase__="[MASK]"				, lowerCAmelCase__=True				, lowerCAmelCase__=None				, **lowerCAmelCase__				, )					->				Optional[Any]:




        '''simple docstring'''
        super().__init__(
            __snake_case				, tokenizer_file=__snake_case				, do_lower_case=__snake_case				, unk_token=__snake_case				, sep_token=__snake_case				, pad_token=__snake_case				, cls_token=__snake_case				, mask_token=__snake_case				, tokenize_chinese_chars=__snake_case				, strip_accents=__snake_case				, **__snake_case				, )

        lowercase__:   List[Any]  			=   json.loads(self.backend_tokenizer.normalizer.__getstate__() )
        if (
            pre_tok_state.get('lowercase'				, __snake_case ) != do_lower_case
            or pre_tok_state.get('strip_accents'				, __snake_case ) != strip_accents
        ):
               lowercase__:   Union[str, Any]  			=   getattr(__snake_case				, pre_tok_state.pop('type' ) )
               lowercase__:   List[Any]  			=   do_lower_case
               lowercase__:   Union[str, Any]  			=   strip_accents
               lowercase__:   int  			=   pre_tok_class(**__snake_case )

        lowercase__:   Optional[int]  			=   do_lower_case
 def __getstate__(   self )					->				List[Any]:




        '''simple docstring'''
        lowercase__:   Tuple  			=   self.__dict__.copy()
        lowercase__:   Dict  			=   BertPreTokenizer()
        return state
 def __setstate__(   self				, lowerCAmelCase__ )					->				Union[str, Any]:




        '''simple docstring'''
        lowercase__:   str  			=   d
        lowercase__:   Dict  			=   self.__dict__['_tokenizer'].get_vocab()
        lowercase__:   int  			=   PreTokenizer.custom(JiebaPreTokenizer(__snake_case ) )
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__=None )					->				Union[str, Any]:




        '''simple docstring'''
        lowercase__:   Union[str, Any]  			=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

        return output
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None )					->				List[int]:




        '''simple docstring'''
        lowercase__:   Dict  			=   [self.sep_token_id]
        lowercase__:   Union[str, Any]  			=   [self.cls_token_id]
        if token_ids_a is None:
               return len(cls + token_ids_a + sep ) * [0]
        return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None )					->				Tuple[str]:




        '''simple docstring'''
        lowercase__:   Optional[int]  			=   self._tokenizer.model.save(__snake_case				, name=__snake_case )
        return tuple(__snake_case )
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__=None				, lowerCAmelCase__=None				, lowerCAmelCase__=False				, **lowerCAmelCase__				, )					->				Any:




        '''simple docstring'''
        lowercase__:   str  			=   BertPreTokenizer()
        return super().save_pretrained(__snake_case				, __snake_case				, __snake_case				, __snake_case				, **__snake_case )





_lowerCAmelCase					:			Optional[int]  				=			"\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n"

_lowerCAmelCase					:			Tuple  				=			[{"type": "code", "content": INSTALL_CONTENT}]
_lowerCAmelCase					:			Optional[int]  				=			{
    "{processor_class}": "FakeProcessorClass",
    "{model_class}": "FakeModelClass",
    "{object_class}": "FakeObjectClass",
}






