



'''simple docstring'''
import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 							lowercase__	(							lowercase						,					unittest.TestCase   ):
	lowercase__  				=			MgpstrTokenizer
	lowercase__  				=			False
	lowercase__  				=			{}
	lowercase__  				=			False





	def 	UpperCamelCase_  ( self					:				List[str] ):







							'''simple docstring'''
							super().setUp()

							# fmt: off
							_UpperCamelCase					:		Optional[Any] 				=   ['[GO]', '[s]', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
							# fmt: on
							_UpperCamelCase					:		str 				=   dict(zip(lowerCamelCase__ ,range(len(lowerCamelCase__ ) ) ) )

							_UpperCamelCase					:		List[str] 				=   os.path.join(self.tmpdirname ,VOCAB_FILES_NAMES['vocab_file'] )
							with open(self.vocab_file ,'w' ,encoding='utf-8' ) as fp:
													fp.write(json.dumps(lowerCamelCase__ ) + '\n' )





	def 	UpperCamelCase_  ( self					:				int ,**lowerCamelCase__					:				str ):







							'''simple docstring'''
							return MgpstrTokenizer.from_pretrained(self.tmpdirname ,**lowerCamelCase__ )





	def 	UpperCamelCase_  ( self					:				str ,lowerCamelCase__					:				Any ):







							'''simple docstring'''
							_UpperCamelCase					:		Any 				=   'tester'
							_UpperCamelCase					:		Optional[int] 				=   'tester'
							return input_text, output_text





	@unittest.skip('MGP-STR always lower cases letters.' )
	def 	UpperCamelCase_  ( self					:				Union[str, Any] ):







							'''simple docstring'''
							pass





	def 	UpperCamelCase_  ( self					:				Optional[int] ):







							'''simple docstring'''
							_UpperCamelCase					:		List[Any] 				=   self.get_tokenizers(do_lower_case=lowerCamelCase__ )
							for tokenizer in tokenizers:
													with self.subTest(F'{tokenizer.__class__.__name__}' ):
																			_UpperCamelCase					:		Union[str, Any] 				=   '[SPECIAL_TOKEN]'

																			tokenizer.add_special_tokens({'cls_token': special_token} )
																			_UpperCamelCase					:		List[Any] 				=   tokenizer.encode([special_token] ,add_special_tokens=lowerCamelCase__ )
																			self.assertEqual(len(lowerCamelCase__ ) ,1 )

																			_UpperCamelCase					:		Any 				=   tokenizer.decode(lowerCamelCase__ ,skip_special_tokens=lowerCamelCase__ )
																			self.assertTrue(special_token not in decoded )





	def 	UpperCamelCase_  ( self					:				Any ):







							'''simple docstring'''
							_UpperCamelCase					:		Optional[int] 				=   self.get_tokenizers()
							for tokenizer in tokenizers:
													with self.subTest(F'{tokenizer.__class__.__name__}' ):
																			_UpperCamelCase				,			_UpperCamelCase					:		List[Any] 				=   self.get_input_output_texts(lowerCamelCase__ )

																			_UpperCamelCase					:		List[str] 				=   tokenizer.tokenize(lowerCamelCase__ )
																			_UpperCamelCase					:		str 				=   tokenizer.convert_tokens_to_ids(lowerCamelCase__ )
																			_UpperCamelCase					:		List[str] 				=   tokenizer.encode(lowerCamelCase__ ,add_special_tokens=lowerCamelCase__ )
																			self.assertListEqual(lowerCamelCase__ ,lowerCamelCase__ )

																			_UpperCamelCase					:		Optional[Any] 				=   tokenizer.convert_ids_to_tokens(lowerCamelCase__ )
																			self.assertNotEqual(len(lowerCamelCase__ ) ,0 )
																			_UpperCamelCase					:		Dict 				=   tokenizer.decode(lowerCamelCase__ )
																			self.assertIsInstance(lowerCamelCase__ ,lowerCamelCase__ )

																			self.assertEqual(text_a.replace(' ' ,'' ) ,lowerCamelCase__ )





	@unittest.skip('MGP-STR tokenizer only handles one sequence.' )
	def 	UpperCamelCase_  ( self					:				Any ):







							'''simple docstring'''
							pass





	@unittest.skip('inputs cannot be pretokenized in MgpstrTokenizer' )
	def 	UpperCamelCase_  ( self					:				Any ):







							'''simple docstring'''
							pass









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_a 						=      {
    "configuration_lxmert": ["LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "LxmertConfig"],
    "tokenization_lxmert": ["LxmertTokenizer"],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a 						=      ["LxmertTokenizerFast"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a 						=      [
			    "LxmertEncoder",
			    "LxmertForPreTraining",
			    "LxmertForQuestionAnswering",
			    "LxmertModel",
			    "LxmertPreTrainedModel",
			    "LxmertVisualFeatureEncoder",
			    "LxmertXLayer",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a 						=      [
			    "TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFLxmertForPreTraining",
			    "TFLxmertMainLayer",
			    "TFLxmertModel",
			    "TFLxmertPreTrainedModel",
			    "TFLxmertVisualFeatureEncoder",
			]


if TYPE_CHECKING:
			from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
			from .tokenization_lxmert import LxmertTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_lxmert_fast import LxmertTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_lxmert import (
						    LxmertEncoder,
						    LxmertForPreTraining,
						    LxmertForQuestionAnswering,
						    LxmertModel,
						    LxmertPreTrainedModel,
						    LxmertVisualFeatureEncoder,
						    LxmertXLayer,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_lxmert import (
						    TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFLxmertForPreTraining,
						    TFLxmertMainLayer,
						    TFLxmertModel,
						    TFLxmertPreTrainedModel,
						    TFLxmertVisualFeatureEncoder,
						)

else:
			import sys

			_a 						=      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



