


import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
		from tokenizers import Tokenizer
		from tokenizers.models import WordPiece


class        __lowerCamelCase      (				unittest.TestCase ):


				"""simple docstring"""







				def        lowerCAmelCase__			(				self     :    Tuple	,							lowerCamelCase_     :    Dict					):
								_lowerCAmelCase  						=list(tokenizer_class.max_model_input_sizes.keys()					)
								for model_name in sa_models[:1]:
												_lowerCAmelCase  						=tokenizer_class.from_pretrained(lowerCamelCase_					)
												self.assertIsNotNone(lowerCamelCase_					)
												self.assertIsInstance(lowerCamelCase_	,							lowerCamelCase_					)
												self.assertIsInstance(lowerCamelCase_	,							lowerCamelCase_					)

												for special_tok in tokenizer.all_special_tokens:
																self.assertIsInstance(lowerCamelCase_	,							lowerCamelCase_					)
																_lowerCAmelCase  						=tokenizer.convert_tokens_to_ids(lowerCamelCase_					)
																self.assertIsInstance(lowerCamelCase_	,							lowerCamelCase_					)






				def        lowerCAmelCase__			(				self     :    Any	,							lowerCamelCase_     :    BatchEncoding	,							lowerCamelCase_     :    Optional[Callable] = None					):
								_lowerCAmelCase  						=pickle.dumps(lowerCamelCase_					)
								self.assertIsNotNone(lowerCamelCase_					)

								_lowerCAmelCase  						=pickle.loads(lowerCamelCase_					)

								# Ensure is_fast is correctly restored
								self.assertEqual(be_restored.is_fast	,							be_original.is_fast					)

								# Ensure encodings are potentially correctly restored
								if be_original.is_fast:
												self.assertIsNotNone(be_restored.encodings					)
								else:
												self.assertIsNone(be_restored.encodings					)

								# Ensure the keys are the same
								for original_v, restored_v in zip(be_original.values()	,							be_restored.values()					):
												if equal_op:
																self.assertTrue(equal_op(lowerCamelCase_	,							lowerCamelCase_					)					)
												else:
																self.assertEqual(lowerCamelCase_	,							lowerCamelCase_					)






				@slow
				def        lowerCAmelCase__			(				self     :    Optional[Any]					):
								self.check_tokenizer_from_pretrained(lowerCamelCase_					)






				def        lowerCAmelCase__			(				self     :    List[str]					):
								self.assertEqual(TensorType("""tf"""					)	,							TensorType.TENSORFLOW					)
								self.assertEqual(TensorType("""pt"""					)	,							TensorType.PYTORCH					)
								self.assertEqual(TensorType("""np"""					)	,							TensorType.NUMPY					)






				@require_tokenizers
				def        lowerCAmelCase__			(				self     :    Tuple					):
								import numpy as np

								_lowerCAmelCase  						=BertTokenizer.from_pretrained("""bert-base-cased"""					)
								_lowerCAmelCase  						=BertTokenizerFast.from_pretrained("""bert-base-cased"""					)

								# Python no tensor
								with self.subTest("""BatchEncoding (Python, return_tensors=None)"""					):
												self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""					)					)

								with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""					):
												self.assert_dump_and_restore(
												    tokenizer_p("""Small example to encode"""	,							return_tensors=TensorType.NUMPY					)	,							np.array_equal					)

								with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""					):
												self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""					)					)

								with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""					):
												self.assert_dump_and_restore(
												    tokenizer_r("""Small example to encode"""	,							return_tensors=TensorType.NUMPY					)	,							np.array_equal					)






				@require_tf
				@require_tokenizers
				def        lowerCAmelCase__			(				self     :    List[Any]					):
								import tensorflow as tf

								def tf_array_equals(lowerCamelCase_     :    int	,							lowerCamelCase_     :    Optional[int]					):
												return tf.reduce_all(tf.equal(lowerCamelCase_	,							lowerCamelCase_					)					)

								_lowerCAmelCase  						=BertTokenizer.from_pretrained("""bert-base-cased"""					)
								_lowerCAmelCase  						=BertTokenizerFast.from_pretrained("""bert-base-cased"""					)

								with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""					):
												self.assert_dump_and_restore(
												    tokenizer_p("""Small example to encode"""	,							return_tensors=TensorType.TENSORFLOW					)	,							lowerCamelCase_					)

								with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""					):
												self.assert_dump_and_restore(
												    tokenizer_r("""Small example to encode"""	,							return_tensors=TensorType.TENSORFLOW					)	,							lowerCamelCase_					)






				@require_torch
				@require_tokenizers
				def        lowerCAmelCase__			(				self     :    Union[str, Any]					):
								import torch

								_lowerCAmelCase  						=BertTokenizer.from_pretrained("""bert-base-cased"""					)
								_lowerCAmelCase  						=BertTokenizerFast.from_pretrained("""bert-base-cased"""					)

								with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""					):
												self.assert_dump_and_restore(
												    tokenizer_p("""Small example to encode"""	,							return_tensors=TensorType.PYTORCH					)	,							torch.equal					)

								with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""					):
												self.assert_dump_and_restore(
												    tokenizer_r("""Small example to encode"""	,							return_tensors=TensorType.PYTORCH					)	,							torch.equal					)






				@require_tokenizers
				def        lowerCAmelCase__			(				self     :    Optional[Any]					):
								_lowerCAmelCase  						=BertTokenizer.from_pretrained("""bert-base-cased"""					)
								_lowerCAmelCase  						=BertTokenizerFast.from_pretrained("""bert-base-cased"""					)

								with self.subTest("""Python Tokenizer"""					):
												self.assertFalse(tokenizer_p("""Small example to_encode"""					).is_fast					)

								with self.subTest("""Rust Tokenizer"""					):
												self.assertTrue(tokenizer_r("""Small example to_encode"""					).is_fast					)






				@require_tokenizers
				def        lowerCAmelCase__			(				self     :    Dict					):
								_lowerCAmelCase  						=BertTokenizerFast.from_pretrained("""bert-base-cased"""					)
								_lowerCAmelCase  						=tokenizer_r(["""Test""", """\xad""", """test"""]	,							is_split_into_words=lowerCamelCase_					)

								self.assertEqual(encoded.word_to_tokens(0					)	,							TokenSpan(start=1	,							end=2					)					)
								self.assertEqual(encoded.word_to_tokens(1					)	,							lowerCamelCase_					)
								self.assertEqual(encoded.word_to_tokens(2					)	,							TokenSpan(start=2	,							end=3					)					)






				def        lowerCAmelCase__			(				self     :    Union[str, Any]					):
								_lowerCAmelCase  						=BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}					)
								_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""np"""					)
								self.assertEqual(tensor_batch["""inputs"""].shape	,							(2, 3)					)
								self.assertEqual(tensor_batch["""labels"""].shape	,							(2,)					)
								# test converting the converted
								with CaptureStderr() as cs:
												_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""np"""					)
								self.assertFalse(len(cs.err					)	,							msg=F"should have no warning, but got {cs.err}"					)

								_lowerCAmelCase  						=BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}					)
								_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""np"""	,							prepend_batch_axis=lowerCamelCase_					)
								self.assertEqual(tensor_batch["""inputs"""].shape	,							(1, 3)					)
								self.assertEqual(tensor_batch["""labels"""].shape	,							(1,)					)






				@require_torch
				def        lowerCAmelCase__			(				self     :    int					):
								_lowerCAmelCase  						=BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}					)
								_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""pt"""					)
								self.assertEqual(tensor_batch["""inputs"""].shape	,							(2, 3)					)
								self.assertEqual(tensor_batch["""labels"""].shape	,							(2,)					)
								# test converting the converted
								with CaptureStderr() as cs:
												_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""pt"""					)
								self.assertFalse(len(cs.err					)	,							msg=F"should have no warning, but got {cs.err}"					)

								_lowerCAmelCase  						=BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}					)
								_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""pt"""	,							prepend_batch_axis=lowerCamelCase_					)
								self.assertEqual(tensor_batch["""inputs"""].shape	,							(1, 3)					)
								self.assertEqual(tensor_batch["""labels"""].shape	,							(1,)					)






				@require_tf
				def        lowerCAmelCase__			(				self     :    Union[str, Any]					):
								_lowerCAmelCase  						=BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}					)
								_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""tf"""					)
								self.assertEqual(tensor_batch["""inputs"""].shape	,							(2, 3)					)
								self.assertEqual(tensor_batch["""labels"""].shape	,							(2,)					)
								# test converting the converted
								with CaptureStderr() as cs:
												_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""tf"""					)
								self.assertFalse(len(cs.err					)	,							msg=F"should have no warning, but got {cs.err}"					)

								_lowerCAmelCase  						=BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}					)
								_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""tf"""	,							prepend_batch_axis=lowerCamelCase_					)
								self.assertEqual(tensor_batch["""inputs"""].shape	,							(1, 3)					)
								self.assertEqual(tensor_batch["""labels"""].shape	,							(1,)					)






				@require_flax
				def        lowerCAmelCase__			(				self     :    List[Any]					):
								_lowerCAmelCase  						=BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}					)
								_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""jax"""					)
								self.assertEqual(tensor_batch["""inputs"""].shape	,							(2, 3)					)
								self.assertEqual(tensor_batch["""labels"""].shape	,							(2,)					)
								# test converting the converted
								with CaptureStderr() as cs:
												_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""jax"""					)
								self.assertFalse(len(cs.err					)	,							msg=F"should have no warning, but got {cs.err}"					)

								_lowerCAmelCase  						=BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}					)
								_lowerCAmelCase  						=batch.convert_to_tensors(tensor_type="""jax"""	,							prepend_batch_axis=lowerCamelCase_					)
								self.assertEqual(tensor_batch["""inputs"""].shape	,							(1, 3)					)
								self.assertEqual(tensor_batch["""labels"""].shape	,							(1,)					)






				def        lowerCAmelCase__			(				self     :    Tuple					):
								_lowerCAmelCase  						=[{"""input_ids""": np.array([0, 1, 2]					)}, {"""input_ids""": np.array([0, 1, 2, 3]					)}]
								_lowerCAmelCase  						=BertTokenizer.from_pretrained("""bert-base-cased"""					)

								_lowerCAmelCase  						=tokenizer.pad(lowerCamelCase_	,							padding=lowerCamelCase_					)
								self.assertTrue(isinstance(batch["""input_ids"""]	,							np.ndarray					)					)
								self.assertEqual(batch["""input_ids"""].tolist()	,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
								_lowerCAmelCase  						=tokenizer.pad(lowerCamelCase_	,							padding=lowerCamelCase_	,							return_tensors="""np"""					)
								self.assertTrue(isinstance(batch["""input_ids"""]	,							np.ndarray					)					)
								self.assertEqual(batch["""input_ids"""].tolist()	,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)






				@require_torch
				def        lowerCAmelCase__			(				self     :    Union[str, Any]					):
								import torch

								_lowerCAmelCase  						=[{"""input_ids""": torch.tensor([0, 1, 2]					)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]					)}]
								_lowerCAmelCase  						=BertTokenizer.from_pretrained("""bert-base-cased"""					)

								_lowerCAmelCase  						=tokenizer.pad(lowerCamelCase_	,							padding=lowerCamelCase_					)
								self.assertTrue(isinstance(batch["""input_ids"""]	,							torch.Tensor					)					)
								self.assertEqual(batch["""input_ids"""].tolist()	,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
								_lowerCAmelCase  						=tokenizer.pad(lowerCamelCase_	,							padding=lowerCamelCase_	,							return_tensors="""pt"""					)
								self.assertTrue(isinstance(batch["""input_ids"""]	,							torch.Tensor					)					)
								self.assertEqual(batch["""input_ids"""].tolist()	,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)






				@require_tf
				def        lowerCAmelCase__			(				self     :    Optional[Any]					):
								import tensorflow as tf

								_lowerCAmelCase  						=[{"""input_ids""": tf.constant([0, 1, 2]					)}, {"""input_ids""": tf.constant([0, 1, 2, 3]					)}]
								_lowerCAmelCase  						=BertTokenizer.from_pretrained("""bert-base-cased"""					)

								_lowerCAmelCase  						=tokenizer.pad(lowerCamelCase_	,							padding=lowerCamelCase_					)
								self.assertTrue(isinstance(batch["""input_ids"""]	,							tf.Tensor					)					)
								self.assertEqual(batch["""input_ids"""].numpy().tolist()	,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
								_lowerCAmelCase  						=tokenizer.pad(lowerCamelCase_	,							padding=lowerCamelCase_	,							return_tensors="""tf"""					)
								self.assertTrue(isinstance(batch["""input_ids"""]	,							tf.Tensor					)					)
								self.assertEqual(batch["""input_ids"""].numpy().tolist()	,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)






				@require_tokenizers
				def        lowerCAmelCase__			(				self     :    Tuple					):
								_lowerCAmelCase  						=Tokenizer(WordPiece(unk_token="""[UNK]"""					)					)
								PreTrainedTokenizerFast(tokenizer_object=lowerCamelCase_					)






				@require_tokenizers
				def        lowerCAmelCase__			(				self     :    int					):
								_lowerCAmelCase  						=Tokenizer(WordPiece(unk_token="""[UNK]"""					)					)
								with tempfile.TemporaryDirectory() as tmpdirname:
												bert_tokenizer.save(os.path.join(lowerCamelCase_	,							"""tokenizer.json"""					)					)
												PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowerCamelCase_	,							"""tokenizer.json"""					)					)





__SCREAMING_SNAKE_CASE	:							Optional[Any]           =  '''
# Transformers installation
! pip install transformers datasets
# To install from source instead of the last release, comment the command above and uncomment the following one.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

__SCREAMING_SNAKE_CASE	:							Optional[int]           =  [{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
__SCREAMING_SNAKE_CASE	:							Optional[int]           =  {
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}

