






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
				from tokenizers import Tokenizer
				from tokenizers.models import WordPiece







class 				lowerCamelCase_     (	unittest.TestCase     ):
							def  SCREAMING_SNAKE_CASE_ (						self		:					Any							,__lowerCamelCase		:					str					):





								'''simple docstring'''



								a		=			list(tokenizer_class.max_model_input_sizes.keys()					)
								for model_name in sa_models[:1]:
									a		=			tokenizer_class.from_pretrained(__lowerCamelCase					)
									self.assertIsNotNone(__lowerCamelCase					)
									self.assertIsInstance(__lowerCamelCase							,__lowerCamelCase					)
									self.assertIsInstance(__lowerCamelCase							,__lowerCamelCase					)

									for special_tok in tokenizer.all_special_tokens:
										self.assertIsInstance(__lowerCamelCase							,__lowerCamelCase					)
										a		=			tokenizer.convert_tokens_to_ids(__lowerCamelCase					)
										self.assertIsInstance(__lowerCamelCase							,__lowerCamelCase					)
							def  SCREAMING_SNAKE_CASE_ (						self		:					str							,__lowerCamelCase		:					BatchEncoding							,__lowerCamelCase		:					Optional[Callable] = None					):





								'''simple docstring'''



								a		=			pickle.dumps(__lowerCamelCase					)
								self.assertIsNotNone(__lowerCamelCase					)

								a		=			pickle.loads(__lowerCamelCase					)

								# Ensure is_fast is correctly restored
								self.assertEqual(be_restored.is_fast							,be_original.is_fast					)

								# Ensure encodings are potentially correctly restored
								if be_original.is_fast:
									self.assertIsNotNone(be_restored.encodings					)
								else:
									self.assertIsNone(be_restored.encodings					)

								# Ensure the keys are the same
								for original_v, restored_v in zip(be_original.values()							,be_restored.values()					):
									if equal_op:
										self.assertTrue(equal_op(__lowerCamelCase							,__lowerCamelCase					)					)
									else:
										self.assertEqual(__lowerCamelCase							,__lowerCamelCase					)
							@slow
							def  SCREAMING_SNAKE_CASE_ (						self		:					Union[str, Any]					):





								'''simple docstring'''



								self.check_tokenizer_from_pretrained(__lowerCamelCase					)
							def  SCREAMING_SNAKE_CASE_ (						self		:					Tuple					):





								'''simple docstring'''



								self.assertEqual(TensorType('''tf'''					)							,TensorType.TENSORFLOW					)
								self.assertEqual(TensorType('''pt'''					)							,TensorType.PYTORCH					)
								self.assertEqual(TensorType('''np'''					)							,TensorType.NUMPY					)
							@require_tokenizers
							def  SCREAMING_SNAKE_CASE_ (						self		:					List[str]					):





								'''simple docstring'''



								import numpy as np

								a		=			BertTokenizer.from_pretrained('''bert-base-cased'''					)
								a		=			BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

								# Python no tensor
								with self.subTest('''BatchEncoding (Python, return_tensors=None)'''					):
									self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''					)					)

								with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''					):
									self.assert_dump_and_restore(
									    tokenizer_p('''Small example to encode'''							,return_tensors=TensorType.NUMPY					)							,np.array_equal					)

								with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''					):
									self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''					)					)

								with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''					):
									self.assert_dump_and_restore(
									    tokenizer_r('''Small example to encode'''							,return_tensors=TensorType.NUMPY					)							,np.array_equal					)
							@require_tf
							@require_tokenizers
							def  SCREAMING_SNAKE_CASE_ (						self		:					Tuple					):





								'''simple docstring'''



								import tensorflow as tf

								def tf_array_equals(__lowerCamelCase		:					Tuple							,__lowerCamelCase		:					Union[str, Any]					):
									return tf.reduce_all(tf.equal(__lowerCamelCase							,__lowerCamelCase					)					)

								a		=			BertTokenizer.from_pretrained('''bert-base-cased'''					)
								a		=			BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

								with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''					):
									self.assert_dump_and_restore(
									    tokenizer_p('''Small example to encode'''							,return_tensors=TensorType.TENSORFLOW					)							,__lowerCamelCase					)

								with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''					):
									self.assert_dump_and_restore(
									    tokenizer_r('''Small example to encode'''							,return_tensors=TensorType.TENSORFLOW					)							,__lowerCamelCase					)
							@require_torch
							@require_tokenizers
							def  SCREAMING_SNAKE_CASE_ (						self		:					Union[str, Any]					):





								'''simple docstring'''



								import torch

								a		=			BertTokenizer.from_pretrained('''bert-base-cased'''					)
								a		=			BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

								with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''					):
									self.assert_dump_and_restore(
									    tokenizer_p('''Small example to encode'''							,return_tensors=TensorType.PYTORCH					)							,torch.equal					)

								with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''					):
									self.assert_dump_and_restore(
									    tokenizer_r('''Small example to encode'''							,return_tensors=TensorType.PYTORCH					)							,torch.equal					)
							@require_tokenizers
							def  SCREAMING_SNAKE_CASE_ (						self		:					Optional[Any]					):





								'''simple docstring'''



								a		=			BertTokenizer.from_pretrained('''bert-base-cased'''					)
								a		=			BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

								with self.subTest('''Python Tokenizer'''					):
									self.assertFalse(tokenizer_p('''Small example to_encode'''					).is_fast					)

								with self.subTest('''Rust Tokenizer'''					):
									self.assertTrue(tokenizer_r('''Small example to_encode'''					).is_fast					)
							@require_tokenizers
							def  SCREAMING_SNAKE_CASE_ (						self		:					Dict					):





								'''simple docstring'''



								a		=			BertTokenizerFast.from_pretrained('''bert-base-cased'''					)
								a		=			tokenizer_r(['''Test''', '''\xad''', '''test''']							,is_split_into_words=__lowerCamelCase					)

								self.assertEqual(encoded.word_to_tokens(0					)							,TokenSpan(start=1							,end=2					)					)
								self.assertEqual(encoded.word_to_tokens(1					)							,__lowerCamelCase					)
								self.assertEqual(encoded.word_to_tokens(2					)							,TokenSpan(start=2							,end=3					)					)
							def  SCREAMING_SNAKE_CASE_ (						self		:					int					):





								'''simple docstring'''



								a		=			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
								a		=			batch.convert_to_tensors(tensor_type='''np'''					)
								self.assertEqual(tensor_batch['''inputs'''].shape							,(2, 3)					)
								self.assertEqual(tensor_batch['''labels'''].shape							,(2,)					)
								# test converting the converted
								with CaptureStderr() as cs:
									a		=			batch.convert_to_tensors(tensor_type='''np'''					)
								self.assertFalse(len(cs.err					)							,msg=F"""should have no warning, but got {cs.err}"""					)

								a		=			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
								a		=			batch.convert_to_tensors(tensor_type='''np'''							,prepend_batch_axis=__lowerCamelCase					)
								self.assertEqual(tensor_batch['''inputs'''].shape							,(1, 3)					)
								self.assertEqual(tensor_batch['''labels'''].shape							,(1,)					)
							@require_torch
							def  SCREAMING_SNAKE_CASE_ (						self		:					Any					):





								'''simple docstring'''



								a		=			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
								a		=			batch.convert_to_tensors(tensor_type='''pt'''					)
								self.assertEqual(tensor_batch['''inputs'''].shape							,(2, 3)					)
								self.assertEqual(tensor_batch['''labels'''].shape							,(2,)					)
								# test converting the converted
								with CaptureStderr() as cs:
									a		=			batch.convert_to_tensors(tensor_type='''pt'''					)
								self.assertFalse(len(cs.err					)							,msg=F"""should have no warning, but got {cs.err}"""					)

								a		=			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
								a		=			batch.convert_to_tensors(tensor_type='''pt'''							,prepend_batch_axis=__lowerCamelCase					)
								self.assertEqual(tensor_batch['''inputs'''].shape							,(1, 3)					)
								self.assertEqual(tensor_batch['''labels'''].shape							,(1,)					)
							@require_tf
							def  SCREAMING_SNAKE_CASE_ (						self		:					Tuple					):





								'''simple docstring'''



								a		=			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
								a		=			batch.convert_to_tensors(tensor_type='''tf'''					)
								self.assertEqual(tensor_batch['''inputs'''].shape							,(2, 3)					)
								self.assertEqual(tensor_batch['''labels'''].shape							,(2,)					)
								# test converting the converted
								with CaptureStderr() as cs:
									a		=			batch.convert_to_tensors(tensor_type='''tf'''					)
								self.assertFalse(len(cs.err					)							,msg=F"""should have no warning, but got {cs.err}"""					)

								a		=			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
								a		=			batch.convert_to_tensors(tensor_type='''tf'''							,prepend_batch_axis=__lowerCamelCase					)
								self.assertEqual(tensor_batch['''inputs'''].shape							,(1, 3)					)
								self.assertEqual(tensor_batch['''labels'''].shape							,(1,)					)
							@require_flax
							def  SCREAMING_SNAKE_CASE_ (						self		:					Tuple					):





								'''simple docstring'''



								a		=			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
								a		=			batch.convert_to_tensors(tensor_type='''jax'''					)
								self.assertEqual(tensor_batch['''inputs'''].shape							,(2, 3)					)
								self.assertEqual(tensor_batch['''labels'''].shape							,(2,)					)
								# test converting the converted
								with CaptureStderr() as cs:
									a		=			batch.convert_to_tensors(tensor_type='''jax'''					)
								self.assertFalse(len(cs.err					)							,msg=F"""should have no warning, but got {cs.err}"""					)

								a		=			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
								a		=			batch.convert_to_tensors(tensor_type='''jax'''							,prepend_batch_axis=__lowerCamelCase					)
								self.assertEqual(tensor_batch['''inputs'''].shape							,(1, 3)					)
								self.assertEqual(tensor_batch['''labels'''].shape							,(1,)					)
							def  SCREAMING_SNAKE_CASE_ (						self		:					List[Any]					):





								'''simple docstring'''



								a		=			[{'''input_ids''': np.array([0, 1, 2]					)}, {'''input_ids''': np.array([0, 1, 2, 3]					)}]
								a		=			BertTokenizer.from_pretrained('''bert-base-cased'''					)

								a		=			tokenizer.pad(__lowerCamelCase							,padding=__lowerCamelCase					)
								self.assertTrue(isinstance(batch['''input_ids''']							,np.ndarray					)					)
								self.assertEqual(batch['''input_ids'''].tolist()							,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
								a		=			tokenizer.pad(__lowerCamelCase							,padding=__lowerCamelCase							,return_tensors='''np'''					)
								self.assertTrue(isinstance(batch['''input_ids''']							,np.ndarray					)					)
								self.assertEqual(batch['''input_ids'''].tolist()							,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
							@require_torch
							def  SCREAMING_SNAKE_CASE_ (						self		:					Union[str, Any]					):





								'''simple docstring'''



								import torch

								a		=			[{'''input_ids''': torch.tensor([0, 1, 2]					)}, {'''input_ids''': torch.tensor([0, 1, 2, 3]					)}]
								a		=			BertTokenizer.from_pretrained('''bert-base-cased'''					)

								a		=			tokenizer.pad(__lowerCamelCase							,padding=__lowerCamelCase					)
								self.assertTrue(isinstance(batch['''input_ids''']							,torch.Tensor					)					)
								self.assertEqual(batch['''input_ids'''].tolist()							,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
								a		=			tokenizer.pad(__lowerCamelCase							,padding=__lowerCamelCase							,return_tensors='''pt'''					)
								self.assertTrue(isinstance(batch['''input_ids''']							,torch.Tensor					)					)
								self.assertEqual(batch['''input_ids'''].tolist()							,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
							@require_tf
							def  SCREAMING_SNAKE_CASE_ (						self		:					Optional[int]					):





								'''simple docstring'''



								import tensorflow as tf

								a		=			[{'''input_ids''': tf.constant([0, 1, 2]					)}, {'''input_ids''': tf.constant([0, 1, 2, 3]					)}]
								a		=			BertTokenizer.from_pretrained('''bert-base-cased'''					)

								a		=			tokenizer.pad(__lowerCamelCase							,padding=__lowerCamelCase					)
								self.assertTrue(isinstance(batch['''input_ids''']							,tf.Tensor					)					)
								self.assertEqual(batch['''input_ids'''].numpy().tolist()							,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
								a		=			tokenizer.pad(__lowerCamelCase							,padding=__lowerCamelCase							,return_tensors='''tf'''					)
								self.assertTrue(isinstance(batch['''input_ids''']							,tf.Tensor					)					)
								self.assertEqual(batch['''input_ids'''].numpy().tolist()							,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
							@require_tokenizers
							def  SCREAMING_SNAKE_CASE_ (						self		:					int					):





								'''simple docstring'''



								a		=			Tokenizer(WordPiece(unk_token='''[UNK]'''					)					)
								PreTrainedTokenizerFast(tokenizer_object=__lowerCamelCase					)




							@require_tokenizers
							def  SCREAMING_SNAKE_CASE_ (						self		:					str					):





								'''simple docstring'''



								a		=			Tokenizer(WordPiece(unk_token='''[UNK]'''					)					)
								with tempfile.TemporaryDirectory() as tmpdirname:
									bert_tokenizer.save(os.path.join(__lowerCamelCase							,'''tokenizer.json'''					)					)
									PreTrainedTokenizerFast(tokenizer_file=os.path.join(__lowerCamelCase							,'''tokenizer.json'''					)					)













import numpy as np





def   SCREAMING_SNAKE_CASE__						(   snake_case_,  snake_case_="utf-8",  snake_case_="surrogatepass"  )   ->  List[str]:


	"""simple docstring"""



	a		=			bin(int.from_bytes(text.encode(snake_case_,  snake_case_  ),  '''big'''  )  )[2:]
	return bits.zfill(8 * ((len(snake_case_  ) + 7) // 8)  )





def   SCREAMING_SNAKE_CASE__						(   snake_case_,  snake_case_="utf-8",  snake_case_="surrogatepass"  )   ->  int:


	"""simple docstring"""



	a		=			int(snake_case_,  2  )
	return n.to_bytes((n.bit_length() + 7) // 8,  '''big'''  ).decode(snake_case_,  snake_case_  ) or "\0"





def   SCREAMING_SNAKE_CASE__						(   snake_case_,  snake_case_  )   ->  List[Any]:


	"""simple docstring"""



	if size_par + len(snake_case_  ) <= 2**size_par - (len(snake_case_  ) - 1):
		raise ValueError('''size of parity don\'t match with size of data'''  )

	a		=			[]
	a		=			[]
	a		=			[bin(snake_case_  )[2:] for x in range(1,  size_par + len(snake_case_  ) + 1  )]

	# sorted information data for the size of the output data
	a		=			[]
	# data position template + parity
	a		=			[]
	# parity bit counter
	a		=			0
	# counter position of data bits
	a		=			0

	for x in range(1,  size_par + len(snake_case_  ) + 1  ):
		# Performs a template of bit positions - who should be given,
		# and who should be parity
		if qtd_bp < size_par:
			if (np.log(snake_case_  ) / np.log(2  )).is_integer():
				data_out_gab.append('''P'''  )
				a		=			qtd_bp + 1
			else:
				data_out_gab.append('''D'''  )
		else:
			data_out_gab.append('''D'''  )

		# Sorts the data to the new output size
		if data_out_gab[-1] == "D":
			data_ord.append(data[cont_data]  )
			cont_data += 1
		else:
			data_ord.append(snake_case_  )

    # Calculates parity
	a		=			0  # parity bit counter
	for bp in range(1,  size_par + 1  ):
		# Bit counter one for a given parity
		a		=			0
		# counter to control the loop reading
		a		=			0
		for x in data_ord:
			if x is not None:
				try:
					a		=			(bin_pos[cont_loop])[-1 * (bp)]
				except IndexError:
					a		=			'''0'''
				if aux == "1" and x == "1":
					cont_bo += 1
			cont_loop += 1
		parity.append(cont_bo % 2  )

		qtd_bp += 1

	# Mount the message
	a		=			0  # parity bit counter
	for x in range(0,  size_par + len(snake_case_  )  ):
		if data_ord[x] is None:
			data_out.append(str(parity[cont_bp]  )  )
			cont_bp += 1
		else:
			data_out.append(data_ord[x]  )

	return data_out





def   SCREAMING_SNAKE_CASE__						(   snake_case_,  snake_case_  )   ->  str:


	"""simple docstring"""



	a		=			[]
	# Parity bit counter
	a		=			0
	# Counter p data bit reading
	a		=			0
	# list of parity received
	a		=			[]
	a		=			[]

	for x in range(1,  len(snake_case_  ) + 1  ):
		# Performs a template of bit positions - who should be given,
		#  and who should be parity
		if qtd_bp < size_par and (np.log(snake_case_  ) / np.log(2  )).is_integer():
			data_out_gab.append('''P'''  )
			a		=			qtd_bp + 1
		else:
			data_out_gab.append('''D'''  )

		# Sorts the data to the new output size
		if data_out_gab[-1] == "D":
			data_output.append(data[cont_data]  )
		else:
			parity_received.append(data[cont_data]  )
		cont_data += 1

	# -----------calculates the parity with the data
	a		=			[]
	a		=			[]
	a		=			[bin(snake_case_  )[2:] for x in range(1,  size_par + len(snake_case_  ) + 1  )]

	#  sorted information data for the size of the output data
	a		=			[]
	# Data position feedback + parity
	a		=			[]
	# Parity bit counter
	a		=			0
	# Counter p data bit reading
	a		=			0

	for x in range(1,  size_par + len(snake_case_  ) + 1  ):
		# Performs a template position of bits - who should be given,
		# and who should be parity
		if qtd_bp < size_par and (np.log(snake_case_  ) / np.log(2  )).is_integer():
			data_out_gab.append('''P'''  )
			a		=			qtd_bp + 1
		else:
			data_out_gab.append('''D'''  )

		# Sorts the data to the new output size
		if data_out_gab[-1] == "D":
			data_ord.append(data_output[cont_data]  )
			cont_data += 1
		else:
			data_ord.append(snake_case_  )

    # Calculates parity
	a		=			0  # parity bit counter
	for bp in range(1,  size_par + 1  ):
		# Bit counter one for a certain parity
		a		=			0
		# Counter to control loop reading
		a		=			0
		for x in data_ord:
			if x is not None:
				try:
					a		=			(bin_pos[cont_loop])[-1 * (bp)]
				except IndexError:
					a		=			'''0'''
				if aux == "1" and x == "1":
					cont_bo += 1
			cont_loop += 1
		parity.append(str(cont_bo % 2  )  )

		qtd_bp += 1

	# Mount the message
	a		=			0  # Parity bit counter
	for x in range(0,  size_par + len(snake_case_  )  ):
		if data_ord[x] is None:
			data_out.append(str(parity[cont_bp]  )  )
			cont_bp += 1
		else:
			data_out.append(data_ord[x]  )

	a		=			parity_received == parity
	return data_output, ack





