


import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
		from tokenizers import Tokenizer
		from tokenizers.models import WordPiece





class       _a							(       unittest.TestCase):




	def 							UpperCAmelCase__(      self		:	Tuple	,      _SCREAMING_SNAKE_CASE		:	Optional[int]	)-> Optional[int]:
			lowerCAmelCase__     :    List[str]					  =      list(tokenizer_class.max_model_input_sizes.keys()	)
			for model_name in sa_models[:1]:
					lowerCAmelCase__     :    Any					  =      tokenizer_class.from_pretrained(lowerCamelCase__	)
					self.assertIsNotNone(lowerCamelCase__	)
					self.assertIsInstance(lowerCamelCase__	,      lowerCamelCase__	)
					self.assertIsInstance(lowerCamelCase__	,      lowerCamelCase__	)

					for special_tok in tokenizer.all_special_tokens:
							self.assertIsInstance(lowerCamelCase__	,      lowerCamelCase__	)
							lowerCAmelCase__     :    Optional[int]					  =      tokenizer.convert_tokens_to_ids(lowerCamelCase__	)
							self.assertIsInstance(lowerCamelCase__	,      lowerCamelCase__	)




	def 							UpperCAmelCase__(      self		:	Tuple	,      _SCREAMING_SNAKE_CASE		:	Any	,      _SCREAMING_SNAKE_CASE		:	Union[str, Any] = None	)-> str:
			lowerCAmelCase__     :    Tuple					  =      pickle.dumps(lowerCamelCase__	)
			self.assertIsNotNone(lowerCamelCase__	)

			lowerCAmelCase__     :    Any					  =      pickle.loads(lowerCamelCase__	)

			# Ensure is_fast is correctly restored
			self.assertEqual(be_restored.is_fast	,      be_original.is_fast	)

			# Ensure encodings are potentially correctly restored
			if be_original.is_fast:
					self.assertIsNotNone(be_restored.encodings	)
			else:
					self.assertIsNone(be_restored.encodings	)

			# Ensure the keys are the same
			for original_v, restored_v in zip(be_original.values()	,      be_restored.values()	):
					if equal_op:
							self.assertTrue(equal_op(lowerCamelCase__	,      lowerCamelCase__	)	)
					else:
							self.assertEqual(lowerCamelCase__	,      lowerCamelCase__	)




	@slow
	def 							UpperCAmelCase__(      self		:	Any	)-> Any:
			self.check_tokenizer_from_pretrained(lowerCamelCase__	)




	def 							UpperCAmelCase__(      self		:	Union[str, Any]	)-> Union[str, Any]:
			self.assertEqual(TensorType('''tf'''	)	,      TensorType.TENSORFLOW	)
			self.assertEqual(TensorType('''pt'''	)	,      TensorType.PYTORCH	)
			self.assertEqual(TensorType('''np'''	)	,      TensorType.NUMPY	)




	@require_tokenizers
	def 							UpperCAmelCase__(      self		:	Dict	)-> str:
			import numpy as np

			lowerCAmelCase__     :    int					  =      BertTokenizer.from_pretrained('''bert-base-cased'''	)
			lowerCAmelCase__     :    Dict					  =      BertTokenizerFast.from_pretrained('''bert-base-cased'''	)

			# Python no tensor
			with self.subTest('''BatchEncoding (Python, return_tensors=None)'''	):
					self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''	)	)

			with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''	):
					self.assert_dump_and_restore(
					    tokenizer_p('''Small example to encode'''	,      return_tensors=TensorType.NUMPY	)	,      np.array_equal	)

			with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''	):
					self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''	)	)

			with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''	):
					self.assert_dump_and_restore(
					    tokenizer_r('''Small example to encode'''	,      return_tensors=TensorType.NUMPY	)	,      np.array_equal	)




	@require_tf
	@require_tokenizers
	def 							UpperCAmelCase__(      self		:	List[Any]	)-> Dict:
			import tensorflow as tf

			def tf_array_equals(_SCREAMING_SNAKE_CASE		:	Dict	,      _SCREAMING_SNAKE_CASE		:	Any	):
					return tf.reduce_all(tf.equal(lowerCamelCase__	,      lowerCamelCase__	)	)

			lowerCAmelCase__     :    Optional[int]					  =      BertTokenizer.from_pretrained('''bert-base-cased'''	)
			lowerCAmelCase__     :    str					  =      BertTokenizerFast.from_pretrained('''bert-base-cased'''	)

			with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''	):
					self.assert_dump_and_restore(
					    tokenizer_p('''Small example to encode'''	,      return_tensors=TensorType.TENSORFLOW	)	,      lowerCamelCase__	)

			with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''	):
					self.assert_dump_and_restore(
					    tokenizer_r('''Small example to encode'''	,      return_tensors=TensorType.TENSORFLOW	)	,      lowerCamelCase__	)




	@require_torch
	@require_tokenizers
	def 							UpperCAmelCase__(      self		:	List[str]	)-> Optional[int]:
			import torch

			lowerCAmelCase__     :    List[str]					  =      BertTokenizer.from_pretrained('''bert-base-cased'''	)
			lowerCAmelCase__     :    Dict					  =      BertTokenizerFast.from_pretrained('''bert-base-cased'''	)

			with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''	):
					self.assert_dump_and_restore(
					    tokenizer_p('''Small example to encode'''	,      return_tensors=TensorType.PYTORCH	)	,      torch.equal	)

			with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''	):
					self.assert_dump_and_restore(
					    tokenizer_r('''Small example to encode'''	,      return_tensors=TensorType.PYTORCH	)	,      torch.equal	)




	@require_tokenizers
	def 							UpperCAmelCase__(      self		:	int	)-> Any:
			lowerCAmelCase__     :    List[Any]					  =      BertTokenizer.from_pretrained('''bert-base-cased'''	)
			lowerCAmelCase__     :    List[str]					  =      BertTokenizerFast.from_pretrained('''bert-base-cased'''	)

			with self.subTest('''Python Tokenizer'''	):
					self.assertFalse(tokenizer_p('''Small example to_encode'''	).is_fast	)

			with self.subTest('''Rust Tokenizer'''	):
					self.assertTrue(tokenizer_r('''Small example to_encode'''	).is_fast	)




	@require_tokenizers
	def 							UpperCAmelCase__(      self		:	Optional[Any]	)-> Tuple:
			lowerCAmelCase__     :    Tuple					  =      BertTokenizerFast.from_pretrained('''bert-base-cased'''	)
			lowerCAmelCase__     :    Any					  =      tokenizer_r(['''Test''', '''\xad''', '''test''']	,      is_split_into_words=lowerCamelCase__	)

			self.assertEqual(encoded.word_to_tokens(0	)	,      TokenSpan(start=1	,      end=2	)	)
			self.assertEqual(encoded.word_to_tokens(1	)	,      lowerCamelCase__	)
			self.assertEqual(encoded.word_to_tokens(2	)	,      TokenSpan(start=2	,      end=3	)	)




	def 							UpperCAmelCase__(      self		:	Dict	)-> Tuple:
			lowerCAmelCase__     :    Optional[int]					  =      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}	)
			lowerCAmelCase__     :    List[Any]					  =      batch.convert_to_tensors(tensor_type='''np'''	)
			self.assertEqual(tensor_batch['''inputs'''].shape	,      (2, 3)	)
			self.assertEqual(tensor_batch['''labels'''].shape	,      (2,)	)
			# test converting the converted
			with CaptureStderr() as cs:
					lowerCAmelCase__     :    List[Any]					  =      batch.convert_to_tensors(tensor_type='''np'''	)
			self.assertFalse(len(cs.err	)	,      msg=F'should have no warning, but got {cs.err}'	)

			lowerCAmelCase__     :    List[Any]					  =      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}	)
			lowerCAmelCase__     :    Tuple					  =      batch.convert_to_tensors(tensor_type='''np'''	,      prepend_batch_axis=lowerCamelCase__	)
			self.assertEqual(tensor_batch['''inputs'''].shape	,      (1, 3)	)
			self.assertEqual(tensor_batch['''labels'''].shape	,      (1,)	)




	@require_torch
	def 							UpperCAmelCase__(      self		:	Optional[int]	)-> Union[str, Any]:
			lowerCAmelCase__     :    Tuple					  =      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}	)
			lowerCAmelCase__     :    Union[str, Any]					  =      batch.convert_to_tensors(tensor_type='''pt'''	)
			self.assertEqual(tensor_batch['''inputs'''].shape	,      (2, 3)	)
			self.assertEqual(tensor_batch['''labels'''].shape	,      (2,)	)
			# test converting the converted
			with CaptureStderr() as cs:
					lowerCAmelCase__     :    Optional[int]					  =      batch.convert_to_tensors(tensor_type='''pt'''	)
			self.assertFalse(len(cs.err	)	,      msg=F'should have no warning, but got {cs.err}'	)

			lowerCAmelCase__     :    Union[str, Any]					  =      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}	)
			lowerCAmelCase__     :    Union[str, Any]					  =      batch.convert_to_tensors(tensor_type='''pt'''	,      prepend_batch_axis=lowerCamelCase__	)
			self.assertEqual(tensor_batch['''inputs'''].shape	,      (1, 3)	)
			self.assertEqual(tensor_batch['''labels'''].shape	,      (1,)	)




	@require_tf
	def 							UpperCAmelCase__(      self		:	List[Any]	)-> Union[str, Any]:
			lowerCAmelCase__     :    Optional[Any]					  =      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}	)
			lowerCAmelCase__     :    Optional[int]					  =      batch.convert_to_tensors(tensor_type='''tf'''	)
			self.assertEqual(tensor_batch['''inputs'''].shape	,      (2, 3)	)
			self.assertEqual(tensor_batch['''labels'''].shape	,      (2,)	)
			# test converting the converted
			with CaptureStderr() as cs:
					lowerCAmelCase__     :    List[str]					  =      batch.convert_to_tensors(tensor_type='''tf'''	)
			self.assertFalse(len(cs.err	)	,      msg=F'should have no warning, but got {cs.err}'	)

			lowerCAmelCase__     :    Any					  =      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}	)
			lowerCAmelCase__     :    Optional[int]					  =      batch.convert_to_tensors(tensor_type='''tf'''	,      prepend_batch_axis=lowerCamelCase__	)
			self.assertEqual(tensor_batch['''inputs'''].shape	,      (1, 3)	)
			self.assertEqual(tensor_batch['''labels'''].shape	,      (1,)	)




	@require_flax
	def 							UpperCAmelCase__(      self		:	Tuple	)-> Dict:
			lowerCAmelCase__     :    Optional[Any]					  =      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}	)
			lowerCAmelCase__     :    Any					  =      batch.convert_to_tensors(tensor_type='''jax'''	)
			self.assertEqual(tensor_batch['''inputs'''].shape	,      (2, 3)	)
			self.assertEqual(tensor_batch['''labels'''].shape	,      (2,)	)
			# test converting the converted
			with CaptureStderr() as cs:
					lowerCAmelCase__     :    Union[str, Any]					  =      batch.convert_to_tensors(tensor_type='''jax'''	)
			self.assertFalse(len(cs.err	)	,      msg=F'should have no warning, but got {cs.err}'	)

			lowerCAmelCase__     :    Union[str, Any]					  =      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}	)
			lowerCAmelCase__     :    List[Any]					  =      batch.convert_to_tensors(tensor_type='''jax'''	,      prepend_batch_axis=lowerCamelCase__	)
			self.assertEqual(tensor_batch['''inputs'''].shape	,      (1, 3)	)
			self.assertEqual(tensor_batch['''labels'''].shape	,      (1,)	)




	def 							UpperCAmelCase__(      self		:	List[Any]	)-> List[str]:
			lowerCAmelCase__     :    str					  =      [{"input_ids": np.array([0, 1, 2]	)}, {"input_ids": np.array([0, 1, 2, 3]	)}]
			lowerCAmelCase__     :    List[str]					  =      BertTokenizer.from_pretrained('''bert-base-cased'''	)

			lowerCAmelCase__     :    List[Any]					  =      tokenizer.pad(lowerCamelCase__	,      padding=lowerCamelCase__	)
			self.assertTrue(isinstance(batch['''input_ids''']	,      np.ndarray	)	)
			self.assertEqual(batch['''input_ids'''].tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
			lowerCAmelCase__     :    int					  =      tokenizer.pad(lowerCamelCase__	,      padding=lowerCamelCase__	,      return_tensors='''np'''	)
			self.assertTrue(isinstance(batch['''input_ids''']	,      np.ndarray	)	)
			self.assertEqual(batch['''input_ids'''].tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)




	@require_torch
	def 							UpperCAmelCase__(      self		:	List[str]	)-> Optional[int]:
			import torch

			lowerCAmelCase__     :    Tuple					  =      [{"input_ids": torch.tensor([0, 1, 2]	)}, {"input_ids": torch.tensor([0, 1, 2, 3]	)}]
			lowerCAmelCase__     :    int					  =      BertTokenizer.from_pretrained('''bert-base-cased'''	)

			lowerCAmelCase__     :    str					  =      tokenizer.pad(lowerCamelCase__	,      padding=lowerCamelCase__	)
			self.assertTrue(isinstance(batch['''input_ids''']	,      torch.Tensor	)	)
			self.assertEqual(batch['''input_ids'''].tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
			lowerCAmelCase__     :    Optional[Any]					  =      tokenizer.pad(lowerCamelCase__	,      padding=lowerCamelCase__	,      return_tensors='''pt'''	)
			self.assertTrue(isinstance(batch['''input_ids''']	,      torch.Tensor	)	)
			self.assertEqual(batch['''input_ids'''].tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)




	@require_tf
	def 							UpperCAmelCase__(      self		:	Dict	)-> List[str]:
			import tensorflow as tf

			lowerCAmelCase__     :    Any					  =      [{"input_ids": tf.constant([0, 1, 2]	)}, {"input_ids": tf.constant([0, 1, 2, 3]	)}]
			lowerCAmelCase__     :    Dict					  =      BertTokenizer.from_pretrained('''bert-base-cased'''	)

			lowerCAmelCase__     :    Dict					  =      tokenizer.pad(lowerCamelCase__	,      padding=lowerCamelCase__	)
			self.assertTrue(isinstance(batch['''input_ids''']	,      tf.Tensor	)	)
			self.assertEqual(batch['''input_ids'''].numpy().tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
			lowerCAmelCase__     :    str					  =      tokenizer.pad(lowerCamelCase__	,      padding=lowerCamelCase__	,      return_tensors='''tf'''	)
			self.assertTrue(isinstance(batch['''input_ids''']	,      tf.Tensor	)	)
			self.assertEqual(batch['''input_ids'''].numpy().tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)




	@require_tokenizers
	def 							UpperCAmelCase__(      self		:	int	)-> Dict:
			lowerCAmelCase__     :    Union[str, Any]					  =      Tokenizer(WordPiece(unk_token='''[UNK]'''	)	)
			PreTrainedTokenizerFast(tokenizer_object=lowerCamelCase__	)




	@require_tokenizers
	def 							UpperCAmelCase__(      self		:	int	)-> List[Any]:
			lowerCAmelCase__     :    List[str]					  =      Tokenizer(WordPiece(unk_token='''[UNK]'''	)	)
			with tempfile.TemporaryDirectory() as tmpdirname:
					bert_tokenizer.save(os.path.join(lowerCamelCase__	,      '''tokenizer.json'''	)	)
					PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowerCamelCase__	,      '''tokenizer.json'''	)	)





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


lowerCamelCase		=	{
    '''configuration_segformer''': ['''SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''SegformerConfig''', '''SegformerOnnxConfig''']
}

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase		=	['''SegformerFeatureExtractor''']
		lowerCamelCase		=	['''SegformerImageProcessor''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase		=	[
		    '''SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''SegformerDecodeHead''',
		    '''SegformerForImageClassification''',
		    '''SegformerForSemanticSegmentation''',
		    '''SegformerLayer''',
		    '''SegformerModel''',
		    '''SegformerPreTrainedModel''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase		=	[
		    '''TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFSegformerDecodeHead''',
		    '''TFSegformerForImageClassification''',
		    '''TFSegformerForSemanticSegmentation''',
		    '''TFSegformerModel''',
		    '''TFSegformerPreTrainedModel''',
		]


if TYPE_CHECKING:
		from .configuration_segformer import SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, SegformerConfig, SegformerOnnxConfig

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_segformer import SegformerFeatureExtractor
				from .image_processing_segformer import SegformerImageProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_segformer import (
				    SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    SegformerDecodeHead,
				    SegformerForImageClassification,
				    SegformerForSemanticSegmentation,
				    SegformerLayer,
				    SegformerModel,
				    SegformerPreTrainedModel,
				)
		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_segformer import (
				    TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFSegformerDecodeHead,
				    TFSegformerForImageClassification,
				    TFSegformerForSemanticSegmentation,
				    TFSegformerModel,
				    TFSegformerPreTrainedModel,
				)

else:
		import sys

		lowerCamelCase		=	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




