



'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__:   Any								=			{
    "configuration_bert": ["BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "BertConfig", "BertOnnxConfig"],
    "tokenization_bert": ["BasicTokenizer", "BertTokenizer", "WordpieceTokenizer"],
}

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   List[str]								=			["BertTokenizerFast"]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   Optional[Any]								=			[
						    "BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "BertForMaskedLM",
						    "BertForMultipleChoice",
						    "BertForNextSentencePrediction",
						    "BertForPreTraining",
						    "BertForQuestionAnswering",
						    "BertForSequenceClassification",
						    "BertForTokenClassification",
						    "BertLayer",
						    "BertLMHeadModel",
						    "BertModel",
						    "BertPreTrainedModel",
						    "load_tf_weights_in_bert",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   List[str]								=			[
						    "TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "TFBertEmbeddings",
						    "TFBertForMaskedLM",
						    "TFBertForMultipleChoice",
						    "TFBertForNextSentencePrediction",
						    "TFBertForPreTraining",
						    "TFBertForQuestionAnswering",
						    "TFBertForSequenceClassification",
						    "TFBertForTokenClassification",
						    "TFBertLMHeadModel",
						    "TFBertMainLayer",
						    "TFBertModel",
						    "TFBertPreTrainedModel",
						]
try:
						if not is_tensorflow_text_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   Any								=			["TFBertTokenizer"]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   Any								=			[
						    "FlaxBertForCausalLM",
						    "FlaxBertForMaskedLM",
						    "FlaxBertForMultipleChoice",
						    "FlaxBertForNextSentencePrediction",
						    "FlaxBertForPreTraining",
						    "FlaxBertForQuestionAnswering",
						    "FlaxBertForSequenceClassification",
						    "FlaxBertForTokenClassification",
						    "FlaxBertModel",
						    "FlaxBertPreTrainedModel",
						]

if TYPE_CHECKING:
						from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
						from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_bert_fast import BertTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_bert import (
												    BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    BertForMaskedLM,
												    BertForMultipleChoice,
												    BertForNextSentencePrediction,
												    BertForPreTraining,
												    BertForQuestionAnswering,
												    BertForSequenceClassification,
												    BertForTokenClassification,
												    BertLayer,
												    BertLMHeadModel,
												    BertModel,
												    BertPreTrainedModel,
												    load_tf_weights_in_bert,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_bert import (
												    TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFBertEmbeddings,
												    TFBertForMaskedLM,
												    TFBertForMultipleChoice,
												    TFBertForNextSentencePrediction,
												    TFBertForPreTraining,
												    TFBertForQuestionAnswering,
												    TFBertForSequenceClassification,
												    TFBertForTokenClassification,
												    TFBertLMHeadModel,
												    TFBertMainLayer,
												    TFBertModel,
												    TFBertPreTrainedModel,
												)

						try:
												if not is_tensorflow_text_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_bert_tf import TFBertTokenizer

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_flax_bert import (
												    FlaxBertForCausalLM,
												    FlaxBertForMaskedLM,
												    FlaxBertForMultipleChoice,
												    FlaxBertForNextSentencePrediction,
												    FlaxBertForPreTraining,
												    FlaxBertForQuestionAnswering,
												    FlaxBertForSequenceClassification,
												    FlaxBertForTokenClassification,
												    FlaxBertModel,
												    FlaxBertPreTrainedModel,
												)

else:
						import sys

						UpperCamelCase__:   Any								=			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)







import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 						lowercase__  (						_UpperCAmelCase     ,       unittest.TestCase  ):
	A__    :   Dict				       =LxmertTokenizer
	A__    :   List[Any]				       =LxmertTokenizerFast
	A__    :   Any				       =True
	A__    :   List[Any]				       =True
	def      A_				(			self							:   Optional[Any]							):
						super().setUp()

						SCREAMING_SNAKE_CASE__													= [
						    '[UNK]',
						    '[CLS]',
						    '[SEP]',
						    'want',
						    '##want',
						    '##ed',
						    'wa',
						    'un',
						    'runn',
						    '##ing',
						    ',',
						    'low',
						    'lowest',
						]
						SCREAMING_SNAKE_CASE__													= os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES['vocab_file']							)
						with open(self.vocab_file       ,   'w'       ,   encoding='utf-8'							) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]							)							)
	def      A_				(			self							:   int       ,   UpperCAmelCase_							:   int							):
						SCREAMING_SNAKE_CASE__													= 'UNwant\u00E9d,running'
						SCREAMING_SNAKE_CASE__													= 'unwanted, running'
						return input_text, output_text
	def      A_				(			self							:   Optional[Any]							):
						SCREAMING_SNAKE_CASE__													= self.tokenizer_class(self.vocab_file							)

						SCREAMING_SNAKE_CASE__													= tokenizer.tokenize('UNwant\u00E9d,running'							)
						self.assertListEqual(UpperCAmelCase_       ,   ['un', '##want', '##ed', ',', 'runn', '##ing']							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)       ,   [7, 4, 5, 10, 8, 9]							)





	def      A_				(			self							:   List[str]							):
						if not self.test_rust_tokenizer:
											return

						SCREAMING_SNAKE_CASE__													= self.get_tokenizer()
						SCREAMING_SNAKE_CASE__													= self.get_rust_tokenizer()

						SCREAMING_SNAKE_CASE__													= 'I was born in 92000, and this is falsé.'

						SCREAMING_SNAKE_CASE__													= tokenizer.tokenize(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= rust_tokenizer.tokenize(UpperCAmelCase_							)
						self.assertListEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= tokenizer.encode(UpperCAmelCase_       ,   add_special_tokens=UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= rust_tokenizer.encode(UpperCAmelCase_       ,   add_special_tokens=UpperCAmelCase_							)
						self.assertListEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= self.get_rust_tokenizer()
						SCREAMING_SNAKE_CASE__													= tokenizer.encode(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= rust_tokenizer.encode(UpperCAmelCase_							)
						self.assertListEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)





