


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


lowerCamelCase											=       logging.get_logger(__name__)

lowerCamelCase											=       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase											=       {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

lowerCamelCase											=       {
    'junnyu/roformer_chinese_small': 15_36,
    'junnyu/roformer_chinese_base': 15_36,
    'junnyu/roformer_chinese_char_small': 5_12,
    'junnyu/roformer_chinese_char_base': 5_12,
    'junnyu/roformer_small_discriminator': 1_28,
    'junnyu/roformer_small_generator': 1_28,
}


lowerCamelCase											=       {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}




class A  (   UpperCamelCase_     ):
      UpperCamelCase__							:  Any      =VOCAB_FILES_NAMES
      UpperCamelCase__							:  Tuple      =PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__							:  List[Any]      =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__							:  Dict      =PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase__							:  List[Any]      =RoFormerTokenizer






      def __init__(					self						:     Any			,   lowercase_						:     List[str]=None			,   lowercase_						:     List[str]=None			,   lowercase_						:     int=True			,   lowercase_						:     str="[UNK]"			,   lowercase_						:     List[Any]="[SEP]"			,   lowercase_						:     str="[PAD]"			,   lowercase_						:     str="[CLS]"			,   lowercase_						:     Optional[int]="[MASK]"			,   lowercase_						:     Optional[int]=True			,   lowercase_						:     int=None			,   **lowercase_						:     Any			,   )					->						Union[str, Any]:
             """simple docstring"""




             super().__init__(
                 lowercase_			,   tokenizer_file=lowercase_			,   do_lower_case=lowercase_			,   unk_token=lowercase_			,   sep_token=lowercase_			,   pad_token=lowercase_			,   cls_token=lowercase_			,   mask_token=lowercase_			,   tokenize_chinese_chars=lowercase_			,   strip_accents=lowercase_			,   **lowercase_			,   )

             _lowerCamelCase       :		List[str]		=json.loads(self.backend_tokenizer.normalizer.__getstate__() )
             if (
                 pre_tok_state.get('lowercase'			,   lowercase_ ) != do_lower_case
                 or pre_tok_state.get('strip_accents'			,   lowercase_ ) != strip_accents
             ):
                    _lowerCamelCase       :		List[str]		=getattr(lowercase_			,   pre_tok_state.pop('type' ) )
                    _lowerCamelCase       :		str		=do_lower_case
                    _lowerCamelCase       :		Tuple		=strip_accents
                    _lowerCamelCase       :		Dict		=pre_tok_class(**lowercase_ )

             _lowerCamelCase       :		int		=do_lower_case






      def __getstate__(					self						:     int )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Any		=self.__dict__.copy()
             _lowerCamelCase       :		str		=BertPreTokenizer()
             return state






      def __setstate__(					self						:     Optional[int]			,   lowercase_						:     Tuple )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=d
             _lowerCamelCase       :		Union[str, Any]		=self.__dict__['_tokenizer'].get_vocab()
             _lowerCamelCase       :		Optional[Any]		=PreTokenizer.custom(JiebaPreTokenizer(lowercase_ ) )






      def        lowerCamelCase       (					self						:     List[str]			,   lowercase_						:     Optional[int]			,   lowercase_						:     Any=None )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		str		=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                    output += token_ids_a + [self.sep_token_id]

             return output






      def        lowerCamelCase       (					self						:     List[Any]			,   lowercase_						:     List[int]			,   lowercase_						:     Optional[List[int]] = None )					->						List[int]:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=[self.sep_token_id]
             _lowerCamelCase       :		Union[str, Any]		=[self.cls_token_id]
             if token_ids_a is None:
                    return len(cls + token_ids_a + sep ) * [0]
             return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






      def        lowerCamelCase       (					self						:     Optional[int]			,   lowercase_						:     str			,   lowercase_						:     Optional[str] = None )					->						Tuple[str]:
             """simple docstring"""




             _lowerCamelCase       :		str		=self._tokenizer.model.save(lowercase_			,   name=lowercase_ )
             return tuple(lowercase_ )






      def        lowerCamelCase       (					self						:     Optional[int]			,   lowercase_						:     Any			,   lowercase_						:     List[Any]=None			,   lowercase_						:     Any=None			,   lowercase_						:     Union[str, Any]=False			,   **lowercase_						:     int			,   )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Dict		=BertPreTokenizer()
             return super().save_pretrained(lowercase_			,   lowercase_			,   lowercase_			,   lowercase_			,   **lowercase_ )






import os
import sys
import unittest


lowerCamelCase											=       os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, 'utils'))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
lowerCamelCase											=       os.path.join(git_repo_path, 'src', 'transformers')

lowerCamelCase											=       '\n{0} = None\n'

lowerCamelCase											=       '\nclass {0}(metaclass=DummyObject):\n    _backends = {1}\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, {1})\n'


lowerCamelCase											=       '\ndef {0}(*args, **kwargs):\n    requires_backends({0}, {1})\n'




class A  (   unittest.TestCase     ):






      def        lowerCamelCase       (					self						:     Tuple )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		int		=find_backend('    _import_structure["models.albert"].append("AlbertTokenizerFast")' )
             self.assertIsNone(lowercase_ )

             _lowerCamelCase       :		List[str]		=find_backend('    if not is_tokenizers_available():' )
             self.assertEqual(lowercase_			,   'tokenizers' )

             _lowerCamelCase       :		List[Any]		=find_backend('    if not is_tensorflow_text_available():' )
             self.assertEqual(lowercase_			,   'tensorflow_text' )

             _lowerCamelCase       :		int		=find_backend('    if not (is_sentencepiece_available() and is_tokenizers_available()):' )
             self.assertEqual(lowercase_			,   'sentencepiece_and_tokenizers' )

             _lowerCamelCase       :		Dict		=find_backend(
                 '    if not (is_sentencepiece_available() and is_tensorflow_text_available()):' )
             self.assertEqual(lowercase_			,   'sentencepiece_and_tensorflow_text' )

             _lowerCamelCase       :		List[Any]		=find_backend(
                 '    if not (is_sentencepiece_available() and is_tokenizers_available() and is_vision_available()):' )
             self.assertEqual(lowercase_			,   'sentencepiece_and_tokenizers_and_vision' )






      def        lowerCamelCase       (					self						:     Optional[Any] )					->						List[str]:
             """simple docstring"""




             _lowerCamelCase       :		Union[str, Any]		=read_init()
             # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
             self.assertIn('torch'			,   lowercase_ )
             self.assertIn('tensorflow_text'			,   lowercase_ )
             self.assertIn('sentencepiece_and_tokenizers'			,   lowercase_ )

             # Likewise, we can't assert on the exact content of a key
             self.assertIn('BertModel'			,   objects['torch'] )
             self.assertIn('TFBertModel'			,   objects['tf'] )
             self.assertIn('FlaxBertModel'			,   objects['flax'] )
             self.assertIn('BertModel'			,   objects['torch'] )
             self.assertIn('TFBertTokenizer'			,   objects['tensorflow_text'] )
             self.assertIn('convert_slow_tokenizer'			,   objects['sentencepiece_and_tokenizers'] )






      def        lowerCamelCase       (					self						:     int )					->						Optional[Any]:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=create_dummy_object('CONSTANT'			,   '\'torch\'' )
             self.assertEqual(lowercase_			,   '\nCONSTANT = None\n' )

             _lowerCamelCase       :		Dict		=create_dummy_object('function'			,   '\'torch\'' )
             self.assertEqual(
                 lowercase_			,   '\ndef function(*args, **kwargs):\n    requires_backends(function, \'torch\')\n' )

             _lowerCamelCase       :		Union[str, Any]		='\nclass FakeClass(metaclass=DummyObject):\n    _backends = \'torch\'\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, \'torch\')\n'
             _lowerCamelCase       :		Tuple		=create_dummy_object('FakeClass'			,   '\'torch\'' )
             self.assertEqual(lowercase_			,   lowercase_ )






      def        lowerCamelCase       (					self						:     List[Any] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Dict		='# This file is autogenerated by the command `make fix-copies`, do not edit.\nfrom ..utils import DummyObject, requires_backends\n\n\nCONSTANT = None\n\n\ndef function(*args, **kwargs):\n    requires_backends(function, ["torch"])\n\n\nclass FakeClass(metaclass=DummyObject):\n    _backends = ["torch"]\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, ["torch"])\n'
             _lowerCamelCase       :		Optional[int]		=create_dummy_files({'torch': ['CONSTANT', 'function', 'FakeClass']} )
             self.assertEqual(dummy_files['torch']			,   lowercase_ )


