




from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
		from transformers import (
		    TFAutoModel,
		    TFAutoModelForCausalLM,
		    TFAutoModelForMaskedLM,
		    TFAutoModelForPreTraining,
		    TFAutoModelForQuestionAnswering,
		    TFAutoModelForSeqaSeqLM,
		    TFAutoModelForSequenceClassification,
		    TFAutoModelForTableQuestionAnswering,
		    TFAutoModelForTokenClassification,
		    TFAutoModelWithLMHead,
		    TFBertForMaskedLM,
		    TFBertForPreTraining,
		    TFBertForQuestionAnswering,
		    TFBertForSequenceClassification,
		    TFBertModel,
		    TFFunnelBaseModel,
		    TFFunnelModel,
		    TFGPTaLMHeadModel,
		    TFRobertaForMaskedLM,
		    TFTaForConditionalGeneration,
		    TFTapasForQuestionAnswering,
		)
		from transformers.models.auto.modeling_tf_auto import (
		    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
		    TF_MODEL_FOR_MASKED_LM_MAPPING,
		    TF_MODEL_FOR_PRETRAINING_MAPPING,
		    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
		    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
		    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
		    TF_MODEL_MAPPING,
		)
		from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
		from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
		from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
		from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST
class 		UpperCAmelCase   (				__lowercase			):







						'''simple docstring'''





						lowerCAmelCase_     		=   "new-model"


if is_tf_available():
		class 		UpperCAmelCase   (				__lowercase			):







								'''simple docstring'''





								lowerCAmelCase_     		=   NewModelConfig
@require_tf
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						@slow
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     '''bert-base-cased'''
							snake_case_		    =     AutoConfig.from_pretrained(snake_case_						)
							self.assertIsNotNone(snake_case_						)
							self.assertIsInstance(snake_case_       ,	snake_case_						)

							snake_case_		    =     TFAutoModel.from_pretrained(snake_case_						)
							self.assertIsNotNone(snake_case_						)
							self.assertIsInstance(snake_case_       ,	snake_case_						)






						@slow
						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     '''bert-base-cased'''
							snake_case_		    =     AutoConfig.from_pretrained(snake_case_						)
							self.assertIsNotNone(snake_case_						)
							self.assertIsInstance(snake_case_       ,	snake_case_						)

							snake_case_		    =     TFAutoModelForPreTraining.from_pretrained(snake_case_						)
							self.assertIsNotNone(snake_case_						)
							self.assertIsInstance(snake_case_       ,	snake_case_						)






						@slow
						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		    =     AutoConfig.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)

								snake_case_		    =     TFAutoModelForCausalLM.from_pretrained(snake_case_						)
								snake_case_		    =     TFAutoModelForCausalLM.from_pretrained(snake_case_       ,	output_loading_info=snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)






						@slow
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		    =     AutoConfig.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)

								snake_case_		    =     TFAutoModelWithLMHead.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)






						@slow
						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		    =     AutoConfig.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)

								snake_case_		    =     TFAutoModelForMaskedLM.from_pretrained(snake_case_						)
								snake_case_		    =     TFAutoModelForMaskedLM.from_pretrained(snake_case_       ,	output_loading_info=snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)






						@slow
						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		    =     AutoConfig.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)

								snake_case_		    =     TFAutoModelForSeqaSeqLM.from_pretrained(snake_case_						)
								snake_case_		    =     TFAutoModelForSeqaSeqLM.from_pretrained(snake_case_       ,	output_loading_info=snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)






						@slow
						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							for model_name in ["bert-base-uncased"]:
								snake_case_		    =     AutoConfig.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)

								snake_case_		    =     TFAutoModelForSequenceClassification.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)






						@slow
						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							for model_name in ["bert-base-uncased"]:
								snake_case_		    =     AutoConfig.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)

								snake_case_		    =     TFAutoModelForQuestionAnswering.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)






						@slow
						@require_tensorflow_probability
						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
								snake_case_		    =     AutoConfig.from_pretrained(snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)

								snake_case_		    =     TFAutoModelForTableQuestionAnswering.from_pretrained(snake_case_						)
								snake_case_		    =     TFAutoModelForTableQuestionAnswering.from_pretrained(
								    snake_case_       ,	output_loading_info=snake_case_						)
								self.assertIsNotNone(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     TFAutoModelWithLMHead.from_pretrained(snake_case_						)
							self.assertIsInstance(snake_case_       ,	snake_case_						)
							self.assertEqual(model.num_parameters()       ,	1_44_10						)
							self.assertEqual(model.num_parameters(only_trainable=snake_case_						)       ,	1_44_10						)






						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     TFAutoModelWithLMHead.from_pretrained(snake_case_						)
							self.assertIsInstance(snake_case_       ,	snake_case_						)
							self.assertEqual(model.num_parameters()       ,	1_44_10						)
							self.assertEqual(model.num_parameters(only_trainable=snake_case_						)       ,	1_44_10						)






						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     TFAutoModel.from_pretrained("sgugger/funnel-random-tiny"						)
							self.assertIsInstance(snake_case_       ,	snake_case_						)

							snake_case_		    =     copy.deepcopy(model.config						)
							snake_case_		    =     ['''FunnelBaseModel''']
							snake_case_		    =     TFAutoModel.from_config(snake_case_						)
							self.assertIsInstance(snake_case_       ,	snake_case_						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								model.save_pretrained(snake_case_						)
								snake_case_		    =     TFAutoModel.from_pretrained(snake_case_						)
								self.assertIsInstance(snake_case_       ,	snake_case_						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							try:
								AutoConfig.register("new-model"       ,	snake_case_						)

								snake_case_		    =     [
								    TFAutoModel,
								    TFAutoModelForCausalLM,
								    TFAutoModelForMaskedLM,
								    TFAutoModelForPreTraining,
								    TFAutoModelForQuestionAnswering,
								    TFAutoModelForSequenceClassification,
								    TFAutoModelForTokenClassification,
								]

								for auto_class in auto_classes:
									with self.subTest(auto_class.__name__						):
										# Wrong config class will raise an error
										with self.assertRaises(snake_case_						):
											auto_class.register(snake_case_       ,	snake_case_						)
										auto_class.register(snake_case_       ,	snake_case_						)
										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(snake_case_						):
											auto_class.register(snake_case_       ,	snake_case_						)

										# Now that the config is registered, it can be used as any other config with the auto-API
										snake_case_		    =     BertModelTester(self						).get_config()
										snake_case_		    =     NewModelConfig(**tiny_config.to_dict()						)
										snake_case_		    =     auto_class.from_config(snake_case_						)
										self.assertIsInstance(snake_case_       ,	snake_case_						)

										with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(snake_case_						)
											snake_case_		    =     auto_class.from_pretrained(snake_case_						)
											self.assertIsInstance(snake_case_       ,	snake_case_						)

							finally:
								if "new-model" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["new-model"]
								for mapping in (
								    TF_MODEL_MAPPING,
								    TF_MODEL_FOR_PRETRAINING_MAPPING,
								    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
								    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
								    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
								    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
								    TF_MODEL_FOR_MASKED_LM_MAPPING,
								):
									if NewModelConfig in mapping._extra_content:
										del mapping._extra_content[NewModelConfig]






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    snake_case_       ,	"bert-base is not a local folder and is not a valid model identifier"						):
								snake_case_		    =     TFAutoModel.from_pretrained("bert-base"						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    snake_case_       ,	r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"						):
								snake_case_		    =     TFAutoModel.from_pretrained(snake_case_       ,	revision="aaaaaa"						)






						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    snake_case_       ,	"hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin"       ,	):
								snake_case_		    =     TFAutoModel.from_pretrained("hf-internal-testing/config-no-model"						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(snake_case_       ,	"Use `from_pt=True` to load this model"						):
								snake_case_		    =     TFAutoModel.from_pretrained("hf-internal-testing/tiny-bert-pt-only"						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"						)
							with RequestCounter() as counter:
								snake_case_		    =     TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"						)
								self.assertEqual(counter.get_request_count       ,	0						)
								self.assertEqual(counter.head_request_count       ,	1						)
								self.assertEqual(counter.other_request_count       ,	0						)

							# With a sharded checkpoint
							snake_case_		    =     TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"						)
							with RequestCounter() as counter:
								snake_case_		    =     TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"						)
								self.assertEqual(counter.get_request_count       ,	0						)
								self.assertEqual(counter.head_request_count       ,	1						)
								self.assertEqual(counter.other_request_count       ,	0						)


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


lowercase__       =logging.get_logger(__name__)

lowercase__       ={'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase__       ={
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

lowercase__       ={
    'junnyu/roformer_chinese_small': 1536,
    'junnyu/roformer_chinese_base': 1536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


lowercase__       ={
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}



class 		UpperCamelCase__  (       __lowercase    ):
   _SCREAMING_SNAKE_CASE      :							Optional[Any]	       =      VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE      :							List[Any]	       =      PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE      :							List[str]	       =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE      :							Optional[Any]	       =      PRETRAINED_INIT_CONFIGURATION
   _SCREAMING_SNAKE_CASE      :							Optional[int]	       =      RoFormerTokenizer
   def __init__(self :					List[str]					,				snake_case_ :					Optional[int]=None					,				snake_case_ :					str=None					,				snake_case_ :					Optional[Any]=True					,				snake_case_ :					str="[UNK]"					,				snake_case_ :					Dict="[SEP]"					,				snake_case_ :					Any="[PAD]"					,				snake_case_ :					str="[CLS]"					,				snake_case_ :					List[Any]="[MASK]"					,				snake_case_ :					Any=True					,				snake_case_ :					List[str]=None					,				**snake_case_ :					Optional[int]					,				):
    super().__init__(
        snake_case_					,				tokenizer_file=snake_case_					,				do_lower_case=snake_case_					,				unk_token=snake_case_					,				sep_token=snake_case_					,				pad_token=snake_case_					,				cls_token=snake_case_					,				mask_token=snake_case_					,				tokenize_chinese_chars=snake_case_					,				strip_accents=snake_case_					,				**snake_case_					,				)

    __a :  Optional[int]						=  json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
    if (
        pre_tok_state.get('''lowercase'''					,				snake_case_   ) != do_lower_case
        or pre_tok_state.get('''strip_accents'''					,				snake_case_   ) != strip_accents
    ):
     __a :  List[str]						=  getattr(snake_case_					,				pre_tok_state.pop('''type'''   )   )
     __a :  Optional[Any]						=  do_lower_case
     __a :  Optional[int]						=  strip_accents
     __a :  List[str]						=  pre_tok_class(**snake_case_   )

    __a :  Optional[Any]						=  do_lower_case
   def __getstate__(self :					Union[str, Any]   ):
    __a :  Any						=  self.__dict__.copy()
    __a :  Union[str, Any]						=  BertPreTokenizer()
    return state
   def __setstate__(self :					Tuple					,				snake_case_ :					Optional[Any]   ):
    __a :  Dict						=  d
    __a :  str						=  self.__dict__['''_tokenizer'''].get_vocab()
    __a :  Optional[Any]						=  PreTokenizer.custom(JiebaPreTokenizer(snake_case_   )   )
   def       lowerCAmelCase			(self :					Optional[int]					,				snake_case_ :					List[Any]					,				snake_case_ :					Optional[Any]=None   ):
    __a :  Optional[int]						=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

    return output
   def       lowerCAmelCase			(self :					Optional[int]					,				snake_case_ :					List[int]					,				snake_case_ :					Optional[List[int]] = None   ):
    __a :  int						=  [self.sep_token_id]
    __a :  List[Any]						=  [self.cls_token_id]
    if token_ids_a is None:
     return len(cls + token_ids_a + sep   ) * [0]
    return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]
   def       lowerCAmelCase			(self :					int					,				snake_case_ :					str					,				snake_case_ :					Optional[str] = None   ):
    __a :  Optional[Any]						=  self._tokenizer.model.save(snake_case_					,				name=snake_case_   )
    return tuple(snake_case_   )







   def       lowerCAmelCase			(self :					Dict					,				snake_case_ :					Dict					,				snake_case_ :					Tuple=None					,				snake_case_ :					Optional[Any]=None					,				snake_case_ :					Union[str, Any]=False					,				**snake_case_ :					Tuple					,				):
    __a :  List[str]						=  BertPreTokenizer()
    return super().save_pretrained(snake_case_					,				snake_case_					,				snake_case_					,				snake_case_					,				**snake_case_   )
