





"""simple docstring"""




from manim import *




class 					_a		( _lowerCAmelCase							):






				def 							snake_case      (					self						:       int				)      -> Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase     :   Optional[Any]      =       Rectangle(height=0.5,					width=0.5				)
										_UpperCamelCase     :   List[str]      =       Rectangle(height=0.46,					width=0.46				).set_stroke(width=0				)
										_UpperCamelCase     :   Tuple      =       Rectangle(height=0.25,					width=0.25				)

										_UpperCamelCase     :   str      =       [mem.copy() for i in range(6				)]
										_UpperCamelCase     :   Optional[int]      =       [mem.copy() for i in range(6				)]
										_UpperCamelCase     :   List[Any]      =       VGroup(*lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0				)
										_UpperCamelCase     :   int      =       VGroup(*lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0				)
										_UpperCamelCase     :   Optional[Any]      =       VGroup(lowerCAmelCase__,					lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0				)
										_UpperCamelCase     :   Union[str, Any]      =       Text('''CPU''',					font_size=2_4				)
										_UpperCamelCase     :   Optional[Any]      =       Group(lowerCAmelCase__,					lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0.5,					aligned_edge=lowerCAmelCase__				)
										cpu.move_to([-2.5, -0.5, 0]				)
										self.add(lowerCAmelCase__				)

										_UpperCamelCase     :   Union[str, Any]      =       [mem.copy() for i in range(4				)]
										_UpperCamelCase     :   List[str]      =       VGroup(*lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0				)
										_UpperCamelCase     :   Optional[int]      =       Text('''GPU''',					font_size=2_4				)
										_UpperCamelCase     :   List[str]      =       Group(lowerCAmelCase__,					lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0.5,					aligned_edge=lowerCAmelCase__				)
										gpu.move_to([-1, -1, 0]				)
										self.add(lowerCAmelCase__				)

										_UpperCamelCase     :   str      =       [mem.copy() for i in range(6				)]
										_UpperCamelCase     :   Any      =       VGroup(*lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0				)

										_UpperCamelCase     :   Any      =       Text('''Model''',					font_size=2_4				)
										_UpperCamelCase     :   List[str]      =       Group(lowerCAmelCase__,					lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0.5,					aligned_edge=lowerCAmelCase__				)
										model.move_to([3, -1.0, 0]				)
										self.add(lowerCAmelCase__				)

										_UpperCamelCase     :   str      =       []
										_UpperCamelCase     :   Union[str, Any]      =       []

										for i, rect in enumerate(lowerCAmelCase__				):
																rect.set_stroke(lowerCAmelCase__				)

																_UpperCamelCase     :   Any      =       Rectangle(height=0.46 / 4,					width=0.46 / 3				).set_stroke(width=0.0				).set_fill(lowerCAmelCase__,					opacity=0.7				)

																if i == 0:
																						cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT				),					buff=0.02,					direction=lowerCAmelCase__				)
																						cpu_target.set_x(cpu_target.get_x() + 0.1				)
																elif i == 3:
																						cpu_target.next_to(model_cpu_arr[0],					direction=lowerCAmelCase__,					buff=0.0				)
																else:
																						cpu_target.next_to(model_cpu_arr[i - 1],					direction=lowerCAmelCase__,					buff=0.0				)
																self.add(lowerCAmelCase__				)
																model_cpu_arr.append(lowerCAmelCase__				)

										self.add(*lowerCAmelCase__,					*lowerCAmelCase__				)

										_UpperCamelCase     :   Optional[Any]      =       [meta_mem.copy() for i in range(6				)]
										_UpperCamelCase     :   int      =       [meta_mem.copy() for i in range(6				)]
										_UpperCamelCase     :   Tuple      =       VGroup(*lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0				)
										_UpperCamelCase     :   Any      =       VGroup(*lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0				)
										_UpperCamelCase     :   Any      =       VGroup(lowerCAmelCase__,					lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0				)
										_UpperCamelCase     :   List[Any]      =       Text('''Disk''',					font_size=2_4				)
										_UpperCamelCase     :   int      =       Group(lowerCAmelCase__,					lowerCAmelCase__				).arrange(lowerCAmelCase__,					buff=0.5,					aligned_edge=lowerCAmelCase__				)
										disk.move_to([-4.0, -1.25, 0]				)
										self.add(lowerCAmelCase__,					lowerCAmelCase__				)

										_UpperCamelCase     :   List[str]      =       []

										for i in range(6				):
																_UpperCamelCase     :   str      =       fill.copy().set_fill(lowerCAmelCase__,					opacity=0.8				)
																target.move_to(disk_left_col_base[i]				).scale(0.5				)
																cpu_disk_arr.append(lowerCAmelCase__				)

										self.add(*lowerCAmelCase__				)

										_UpperCamelCase     :   str      =       Square(side_length=2.2				)
										key.move_to([-5, 2, 0]				)

										_UpperCamelCase     :   int      =       MarkupText(
										    f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model""",					font_size=1_8,					)

										key_text.move_to([-5, 2.4, 0]				)

										self.add(lowerCAmelCase__,					lowerCAmelCase__				)

										_UpperCamelCase     :   str      =       MarkupText(
										    f"""<span fgcolor='{BLUE}'>●</span> Checkpoint""",					font_size=1_8,					)

										blue_text.next_to(lowerCAmelCase__,					DOWN * 2.4,					aligned_edge=key_text.get_left()				)
										self.add(lowerCAmelCase__				)

										_UpperCamelCase     :   Any      =       MarkupText(f"""The offloaded weights are all sent to the CPU.""",					font_size=2_4				)
										step_a.move_to([2, 2, 0]				)

										self.play(Write(lowerCAmelCase__,					run_time=3				)				)

										for i in range(6				):
																_UpperCamelCase     :   Any      =       cpu_disk_arr[i]
																_UpperCamelCase     :   Any      =       rect.copy().set_fill(lowerCAmelCase__,					opacity=0.8				).scale(2.0				)
																cpa.generate_target()
																cpa.target.move_to(model_base[i]				)

																if i == 0:
																						rect.set_fill(lowerCAmelCase__,					opacity=0.8				)
																						rect.generate_target()
																						rect.target.move_to(cpu_left_col_base[0]				).scale(2.0				)

																						self.remove(
																						    *lowerCAmelCase__,					*lowerCAmelCase__,					)

																else:
																						rect.generate_target()
																						rect.target.move_to(cpu_left_col_base[i]				).scale(2.0				)
																self.play(MoveToTarget(lowerCAmelCase__				),					MoveToTarget(lowerCAmelCase__				),					model_base[i].animate.set_stroke(lowerCAmelCase__				)				)
										self.play(FadeOut(lowerCAmelCase__				)				)

										_UpperCamelCase     :   Dict      =       MarkupText(
										    f"""Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed.""",					font_size=2_4,					)
										step_a.move_to([2, 2, 0]				)

										self.play(Write(lowerCAmelCase__,					run_time=3				)				)

										_UpperCamelCase     :   Dict      =       []
										_UpperCamelCase     :   Any      =       []
										for i in range(6				):
																_UpperCamelCase     :   Optional[Any]      =       Arrow(start=lowerCAmelCase__,					end=lowerCAmelCase__,					color=lowerCAmelCase__,					buff=0.5				)
																a.next_to(model_base[i].get_left(),					lowerCAmelCase__,					buff=0.2				)
																arrows.append(lowerCAmelCase__				)
																animations.append(Write(lowerCAmelCase__				)				)
										self.play(*lowerCAmelCase__				)
										self.wait()




"""simple docstring"""




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
					from tokenizers import Tokenizer
					from tokenizers.models import WordPiece

class        __SCREAMING_SNAKE_CASE     (   unittest.TestCase							):







				def   lowercase_		(    self    :Optional[Any]      ,__UpperCAmelCase    :Dict  )      ->					Optional[Any]:




									"""simple docstring"""



									lowerCamelCase__		:   Dict  				=					list(tokenizer_class.max_model_input_sizes.keys()  )
									for model_name in sa_models[:1]:
														lowerCamelCase__		:   Tuple  				=					tokenizer_class.from_pretrained(__UpperCAmelCase  )
														self.assertIsNotNone(__UpperCAmelCase  )
														self.assertIsInstance(__UpperCAmelCase      ,__UpperCAmelCase  )
														self.assertIsInstance(__UpperCAmelCase      ,__UpperCAmelCase  )

														for special_tok in tokenizer.all_special_tokens:
																			self.assertIsInstance(__UpperCAmelCase      ,__UpperCAmelCase  )
																			lowerCamelCase__		:   str  				=					tokenizer.convert_tokens_to_ids(__UpperCAmelCase  )
																			self.assertIsInstance(__UpperCAmelCase      ,__UpperCAmelCase  )







				def   lowercase_		(    self    :Any      ,__UpperCAmelCase    :BatchEncoding      ,__UpperCAmelCase    :Optional[Callable] = None  )      ->					Optional[Any]:




									"""simple docstring"""



									lowerCamelCase__		:   List[str]  				=					pickle.dumps(__UpperCAmelCase  )
									self.assertIsNotNone(__UpperCAmelCase  )

									lowerCamelCase__		:   int  				=					pickle.loads(__UpperCAmelCase  )

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast      ,be_original.is_fast  )

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
														self.assertIsNotNone(be_restored.encodings  )
									else:
														self.assertIsNone(be_restored.encodings  )

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()      ,be_restored.values()  ):
														if equal_op:
																			self.assertTrue(equal_op(__UpperCAmelCase      ,__UpperCAmelCase  )  )
														else:
																			self.assertEqual(__UpperCAmelCase      ,__UpperCAmelCase  )







				@slow
				def   lowercase_		(    self    :Tuple  )      ->					List[str]:




									"""simple docstring"""



									self.check_tokenizer_from_pretrained(__UpperCAmelCase  )







				def   lowercase_		(    self    :Optional[int]  )      ->					int:




									"""simple docstring"""



									self.assertEqual(TensorType('''tf'''  )      ,TensorType.TENSORFLOW  )
									self.assertEqual(TensorType('''pt'''  )      ,TensorType.PYTORCH  )
									self.assertEqual(TensorType('''np'''  )      ,TensorType.NUMPY  )







				@require_tokenizers
				def   lowercase_		(    self    :List[Any]  )      ->					Optional[Any]:




									"""simple docstring"""



									import numpy as np

									lowerCamelCase__		:   Any  				=					BertTokenizer.from_pretrained('''bert-base-cased'''  )
									lowerCamelCase__		:   Optional[int]  				=					BertTokenizerFast.from_pretrained('''bert-base-cased'''  )

									# Python no tensor
									with self.subTest('''BatchEncoding (Python, return_tensors=None)'''  ):
														self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''  )  )

									with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''  ):
														self.assert_dump_and_restore(
														    tokenizer_p('''Small example to encode'''      ,return_tensors=TensorType.NUMPY  )      ,np.array_equal  )

									with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''  ):
														self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''  )  )

									with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''  ):
														self.assert_dump_and_restore(
														    tokenizer_r('''Small example to encode'''      ,return_tensors=TensorType.NUMPY  )      ,np.array_equal  )







				@require_tf
				@require_tokenizers
				def   lowercase_		(    self    :Optional[int]  )      ->					int:




									"""simple docstring"""



									import tensorflow as tf

									def tf_array_equals(__UpperCAmelCase    :List[Any]      ,__UpperCAmelCase    :List[Any]  ):
														return tf.reduce_all(tf.equal(__UpperCAmelCase      ,__UpperCAmelCase  )  )

									lowerCamelCase__		:   str  				=					BertTokenizer.from_pretrained('''bert-base-cased'''  )
									lowerCamelCase__		:   List[Any]  				=					BertTokenizerFast.from_pretrained('''bert-base-cased'''  )

									with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''  ):
														self.assert_dump_and_restore(
														    tokenizer_p('''Small example to encode'''      ,return_tensors=TensorType.TENSORFLOW  )      ,__UpperCAmelCase  )

									with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''  ):
														self.assert_dump_and_restore(
														    tokenizer_r('''Small example to encode'''      ,return_tensors=TensorType.TENSORFLOW  )      ,__UpperCAmelCase  )







				@require_torch
				@require_tokenizers
				def   lowercase_		(    self    :Dict  )      ->					Optional[int]:




									"""simple docstring"""



									import torch

									lowerCamelCase__		:   Optional[int]  				=					BertTokenizer.from_pretrained('''bert-base-cased'''  )
									lowerCamelCase__		:   List[str]  				=					BertTokenizerFast.from_pretrained('''bert-base-cased'''  )

									with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''  ):
														self.assert_dump_and_restore(
														    tokenizer_p('''Small example to encode'''      ,return_tensors=TensorType.PYTORCH  )      ,torch.equal  )

									with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''  ):
														self.assert_dump_and_restore(
														    tokenizer_r('''Small example to encode'''      ,return_tensors=TensorType.PYTORCH  )      ,torch.equal  )







				@require_tokenizers
				def   lowercase_		(    self    :Tuple  )      ->					List[str]:




									"""simple docstring"""



									lowerCamelCase__		:   Dict  				=					BertTokenizer.from_pretrained('''bert-base-cased'''  )
									lowerCamelCase__		:   Any  				=					BertTokenizerFast.from_pretrained('''bert-base-cased'''  )

									with self.subTest('''Python Tokenizer'''  ):
														self.assertFalse(tokenizer_p('''Small example to_encode'''  ).is_fast  )

									with self.subTest('''Rust Tokenizer'''  ):
														self.assertTrue(tokenizer_r('''Small example to_encode'''  ).is_fast  )







				@require_tokenizers
				def   lowercase_		(    self    :List[str]  )      ->					str:




									"""simple docstring"""



									lowerCamelCase__		:   Optional[Any]  				=					BertTokenizerFast.from_pretrained('''bert-base-cased'''  )
									lowerCamelCase__		:   str  				=					tokenizer_r(['''Test''', '''\xad''', '''test''']      ,is_split_into_words=__UpperCAmelCase  )

									self.assertEqual(encoded.word_to_tokens(0  )      ,TokenSpan(start=1      ,end=2  )  )
									self.assertEqual(encoded.word_to_tokens(1  )      ,__UpperCAmelCase  )
									self.assertEqual(encoded.word_to_tokens(2  )      ,TokenSpan(start=2      ,end=3  )  )







				def   lowercase_		(    self    :Dict  )      ->					Optional[int]:




									"""simple docstring"""



									lowerCamelCase__		:   Optional[int]  				=					BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}  )
									lowerCamelCase__		:   Tuple  				=					batch.convert_to_tensors(tensor_type='''np'''  )
									self.assertEqual(tensor_batch['''inputs'''].shape      ,(2, 3)  )
									self.assertEqual(tensor_batch['''labels'''].shape      ,(2,)  )
									# test converting the converted
									with CaptureStderr() as cs:
														lowerCamelCase__		:   List[str]  				=					batch.convert_to_tensors(tensor_type='''np'''  )
									self.assertFalse(len(cs.err  )      ,msg=F"""should have no warning, but got {cs.err}"""  )

									lowerCamelCase__		:   Union[str, Any]  				=					BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}  )
									lowerCamelCase__		:   Dict  				=					batch.convert_to_tensors(tensor_type='''np'''      ,prepend_batch_axis=__UpperCAmelCase  )
									self.assertEqual(tensor_batch['''inputs'''].shape      ,(1, 3)  )
									self.assertEqual(tensor_batch['''labels'''].shape      ,(1,)  )







				@require_torch
				def   lowercase_		(    self    :Optional[int]  )      ->					List[Any]:




									"""simple docstring"""



									lowerCamelCase__		:   List[Any]  				=					BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}  )
									lowerCamelCase__		:   Any  				=					batch.convert_to_tensors(tensor_type='''pt'''  )
									self.assertEqual(tensor_batch['''inputs'''].shape      ,(2, 3)  )
									self.assertEqual(tensor_batch['''labels'''].shape      ,(2,)  )
									# test converting the converted
									with CaptureStderr() as cs:
														lowerCamelCase__		:   Optional[int]  				=					batch.convert_to_tensors(tensor_type='''pt'''  )
									self.assertFalse(len(cs.err  )      ,msg=F"""should have no warning, but got {cs.err}"""  )

									lowerCamelCase__		:   Optional[int]  				=					BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}  )
									lowerCamelCase__		:   Optional[Any]  				=					batch.convert_to_tensors(tensor_type='''pt'''      ,prepend_batch_axis=__UpperCAmelCase  )
									self.assertEqual(tensor_batch['''inputs'''].shape      ,(1, 3)  )
									self.assertEqual(tensor_batch['''labels'''].shape      ,(1,)  )







				@require_tf
				def   lowercase_		(    self    :Optional[Any]  )      ->					Union[str, Any]:




									"""simple docstring"""



									lowerCamelCase__		:   Optional[Any]  				=					BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}  )
									lowerCamelCase__		:   Union[str, Any]  				=					batch.convert_to_tensors(tensor_type='''tf'''  )
									self.assertEqual(tensor_batch['''inputs'''].shape      ,(2, 3)  )
									self.assertEqual(tensor_batch['''labels'''].shape      ,(2,)  )
									# test converting the converted
									with CaptureStderr() as cs:
														lowerCamelCase__		:   Optional[Any]  				=					batch.convert_to_tensors(tensor_type='''tf'''  )
									self.assertFalse(len(cs.err  )      ,msg=F"""should have no warning, but got {cs.err}"""  )

									lowerCamelCase__		:   int  				=					BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}  )
									lowerCamelCase__		:   Dict  				=					batch.convert_to_tensors(tensor_type='''tf'''      ,prepend_batch_axis=__UpperCAmelCase  )
									self.assertEqual(tensor_batch['''inputs'''].shape      ,(1, 3)  )
									self.assertEqual(tensor_batch['''labels'''].shape      ,(1,)  )







				@require_flax
				def   lowercase_		(    self    :Optional[int]  )      ->					List[str]:




									"""simple docstring"""



									lowerCamelCase__		:   Optional[Any]  				=					BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}  )
									lowerCamelCase__		:   str  				=					batch.convert_to_tensors(tensor_type='''jax'''  )
									self.assertEqual(tensor_batch['''inputs'''].shape      ,(2, 3)  )
									self.assertEqual(tensor_batch['''labels'''].shape      ,(2,)  )
									# test converting the converted
									with CaptureStderr() as cs:
														lowerCamelCase__		:   Any  				=					batch.convert_to_tensors(tensor_type='''jax'''  )
									self.assertFalse(len(cs.err  )      ,msg=F"""should have no warning, but got {cs.err}"""  )

									lowerCamelCase__		:   List[Any]  				=					BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}  )
									lowerCamelCase__		:   Tuple  				=					batch.convert_to_tensors(tensor_type='''jax'''      ,prepend_batch_axis=__UpperCAmelCase  )
									self.assertEqual(tensor_batch['''inputs'''].shape      ,(1, 3)  )
									self.assertEqual(tensor_batch['''labels'''].shape      ,(1,)  )







				def   lowercase_		(    self    :str  )      ->					Union[str, Any]:




									"""simple docstring"""



									lowerCamelCase__		:   str  				=					[{'''input_ids''': np.array([0, 1, 2]  )}, {'''input_ids''': np.array([0, 1, 2, 3]  )}]
									lowerCamelCase__		:   Optional[Any]  				=					BertTokenizer.from_pretrained('''bert-base-cased'''  )

									lowerCamelCase__		:   Dict  				=					tokenizer.pad(__UpperCAmelCase      ,padding=__UpperCAmelCase  )
									self.assertTrue(isinstance(batch['''input_ids''']      ,np.ndarray  )  )
									self.assertEqual(batch['''input_ids'''].tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
									lowerCamelCase__		:   Optional[Any]  				=					tokenizer.pad(__UpperCAmelCase      ,padding=__UpperCAmelCase      ,return_tensors='''np'''  )
									self.assertTrue(isinstance(batch['''input_ids''']      ,np.ndarray  )  )
									self.assertEqual(batch['''input_ids'''].tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )







				@require_torch
				def   lowercase_		(    self    :Tuple  )      ->					List[str]:




									"""simple docstring"""



									import torch

									lowerCamelCase__		:   Union[str, Any]  				=					[{'''input_ids''': torch.tensor([0, 1, 2]  )}, {'''input_ids''': torch.tensor([0, 1, 2, 3]  )}]
									lowerCamelCase__		:   Any  				=					BertTokenizer.from_pretrained('''bert-base-cased'''  )

									lowerCamelCase__		:   Tuple  				=					tokenizer.pad(__UpperCAmelCase      ,padding=__UpperCAmelCase  )
									self.assertTrue(isinstance(batch['''input_ids''']      ,torch.Tensor  )  )
									self.assertEqual(batch['''input_ids'''].tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
									lowerCamelCase__		:   Optional[int]  				=					tokenizer.pad(__UpperCAmelCase      ,padding=__UpperCAmelCase      ,return_tensors='''pt'''  )
									self.assertTrue(isinstance(batch['''input_ids''']      ,torch.Tensor  )  )
									self.assertEqual(batch['''input_ids'''].tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )







				@require_tf
				def   lowercase_		(    self    :int  )      ->					Optional[int]:




									"""simple docstring"""



									import tensorflow as tf

									lowerCamelCase__		:   List[Any]  				=					[{'''input_ids''': tf.constant([0, 1, 2]  )}, {'''input_ids''': tf.constant([0, 1, 2, 3]  )}]
									lowerCamelCase__		:   str  				=					BertTokenizer.from_pretrained('''bert-base-cased'''  )

									lowerCamelCase__		:   Tuple  				=					tokenizer.pad(__UpperCAmelCase      ,padding=__UpperCAmelCase  )
									self.assertTrue(isinstance(batch['''input_ids''']      ,tf.Tensor  )  )
									self.assertEqual(batch['''input_ids'''].numpy().tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
									lowerCamelCase__		:   Tuple  				=					tokenizer.pad(__UpperCAmelCase      ,padding=__UpperCAmelCase      ,return_tensors='''tf'''  )
									self.assertTrue(isinstance(batch['''input_ids''']      ,tf.Tensor  )  )
									self.assertEqual(batch['''input_ids'''].numpy().tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )







				@require_tokenizers
				def   lowercase_		(    self    :Dict  )      ->					Optional[int]:




									"""simple docstring"""



									lowerCamelCase__		:   List[Any]  				=					Tokenizer(WordPiece(unk_token='''[UNK]'''  )  )
									PreTrainedTokenizerFast(tokenizer_object=__UpperCAmelCase  )







				@require_tokenizers
				def   lowercase_		(    self    :str  )      ->					Union[str, Any]:




									"""simple docstring"""



									lowerCamelCase__		:   int  				=					Tokenizer(WordPiece(unk_token='''[UNK]'''  )  )
									with tempfile.TemporaryDirectory() as tmpdirname:
														bert_tokenizer.save(os.path.join(__UpperCAmelCase      ,'''tokenizer.json'''  )  )
														PreTrainedTokenizerFast(tokenizer_file=os.path.join(__UpperCAmelCase      ,'''tokenizer.json'''  )  )






