

'''simple docstring'''

import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
							from tokenizers import Tokenizer
							from tokenizers.models import WordPiece
class _lowerCamelCase (		unittest.TestCase							):



		'''simple docstring'''







		def      __lowerCAmelCase	(      self	:		Any						,  _A	:		Optional[Any]    )						->				Optional[int]:
			__magic_name__ :					Optional[int]  			=  list(tokenizer_class.max_model_input_sizes.keys()    )
			for model_name in sa_models[:1]:
				__magic_name__ :					List[str]  			=  tokenizer_class.from_pretrained(_A    )
				self.assertIsNotNone(_A    )
				self.assertIsInstance(_A						,  _A    )
				self.assertIsInstance(_A						,  _A    )

				for special_tok in tokenizer.all_special_tokens:
					self.assertIsInstance(_A						,  _A    )
					__magic_name__ :					Optional[Any]  			=  tokenizer.convert_tokens_to_ids(_A    )
					self.assertIsInstance(_A						,  _A    )







		def      __lowerCAmelCase	(      self	:		Optional[Any]						,  _A	:		BatchEncoding						,  _A	:		Optional[Callable] = None    )						->				Tuple:
			__magic_name__ :					Union[str, Any]  			=  pickle.dumps(_A    )
			self.assertIsNotNone(_A    )

			__magic_name__ :					Optional[Any]  			=  pickle.loads(_A    )

			# Ensure is_fast is correctly restored
			self.assertEqual(be_restored.is_fast						,  be_original.is_fast    )

			# Ensure encodings are potentially correctly restored
			if be_original.is_fast:
				self.assertIsNotNone(be_restored.encodings    )
			else:
				self.assertIsNone(be_restored.encodings    )

			# Ensure the keys are the same
			for original_v, restored_v in zip(be_original.values()						,  be_restored.values()    ):
				if equal_op:
					self.assertTrue(equal_op(_A						,  _A    )    )
				else:
					self.assertEqual(_A						,  _A    )







		@slow
		def      __lowerCAmelCase	(      self	:		Dict    )						->				Optional[Any]:
			self.check_tokenizer_from_pretrained(_A    )







		def      __lowerCAmelCase	(      self	:		str    )						->				List[Any]:
			self.assertEqual(TensorType('tf'    )						,  TensorType.TENSORFLOW    )
			self.assertEqual(TensorType('pt'    )						,  TensorType.PYTORCH    )
			self.assertEqual(TensorType('np'    )						,  TensorType.NUMPY    )







		@require_tokenizers
		def      __lowerCAmelCase	(      self	:		List[str]    )						->				Optional[int]:
			import numpy as np

			__magic_name__ :					Dict  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
			__magic_name__ :					str  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )

			# Python no tensor
			with self.subTest('BatchEncoding (Python, return_tensors=None)'    ):
				self.assert_dump_and_restore(tokenizer_p('Small example to encode'    )    )

			with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'    ):
				self.assert_dump_and_restore(
				    tokenizer_p('Small example to encode'						,  return_tensors=TensorType.NUMPY    )						,  np.array_equal    )

			with self.subTest('BatchEncoding (Rust, return_tensors=None)'    ):
				self.assert_dump_and_restore(tokenizer_r('Small example to encode'    )    )

			with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'    ):
				self.assert_dump_and_restore(
				    tokenizer_r('Small example to encode'						,  return_tensors=TensorType.NUMPY    )						,  np.array_equal    )







		@require_tf
		@require_tokenizers
		def      __lowerCAmelCase	(      self	:		List[Any]    )						->				List[str]:
			import tensorflow as tf

			def tf_array_equals(_A	:		str						,  _A	:		Union[str, Any]    ):
				return tf.reduce_all(tf.equal(_A						,  _A    )    )

			__magic_name__ :					str  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
			__magic_name__ :					Optional[Any]  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )

			with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'    ):
				self.assert_dump_and_restore(
				    tokenizer_p('Small example to encode'						,  return_tensors=TensorType.TENSORFLOW    )						,  _A    )

			with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'    ):
				self.assert_dump_and_restore(
				    tokenizer_r('Small example to encode'						,  return_tensors=TensorType.TENSORFLOW    )						,  _A    )







		@require_torch
		@require_tokenizers
		def      __lowerCAmelCase	(      self	:		Any    )						->				List[str]:
			import torch

			__magic_name__ :					Union[str, Any]  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
			__magic_name__ :					Optional[int]  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )

			with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'    ):
				self.assert_dump_and_restore(
				    tokenizer_p('Small example to encode'						,  return_tensors=TensorType.PYTORCH    )						,  torch.equal    )

			with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'    ):
				self.assert_dump_and_restore(
				    tokenizer_r('Small example to encode'						,  return_tensors=TensorType.PYTORCH    )						,  torch.equal    )







		@require_tokenizers
		def      __lowerCAmelCase	(      self	:		List[str]    )						->				Dict:
			__magic_name__ :					Optional[int]  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
			__magic_name__ :					Dict  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )

			with self.subTest('Python Tokenizer'    ):
				self.assertFalse(tokenizer_p('Small example to_encode'    ).is_fast    )

			with self.subTest('Rust Tokenizer'    ):
				self.assertTrue(tokenizer_r('Small example to_encode'    ).is_fast    )







		@require_tokenizers
		def      __lowerCAmelCase	(      self	:		Union[str, Any]    )						->				Optional[int]:
			__magic_name__ :					Optional[int]  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )
			__magic_name__ :					Tuple  			=  tokenizer_r(['Test', '\xad', 'test']						,  is_split_into_words=_A    )

			self.assertEqual(encoded.word_to_tokens(0    )						,  TokenSpan(start=1						,  end=2    )    )
			self.assertEqual(encoded.word_to_tokens(1    )						,  _A    )
			self.assertEqual(encoded.word_to_tokens(2    )						,  TokenSpan(start=2						,  end=3    )    )







		def      __lowerCAmelCase	(      self	:		Optional[int]    )						->				Optional[Any]:
			__magic_name__ :					Any  			=  BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}    )
			__magic_name__ :					Dict  			=  batch.convert_to_tensors(tensor_type='np'    )
			self.assertEqual(tensor_batch['inputs'].shape						,  (2, 3)    )
			self.assertEqual(tensor_batch['labels'].shape						,  (2,)    )
			# test converting the converted
			with CaptureStderr() as cs:
				__magic_name__ :					Any  			=  batch.convert_to_tensors(tensor_type='np'    )
			self.assertFalse(len(cs.err    )						,  msg=F'should have no warning, but got {cs.err}'    )

			__magic_name__ :					Optional[Any]  			=  BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}    )
			__magic_name__ :					List[Any]  			=  batch.convert_to_tensors(tensor_type='np'						,  prepend_batch_axis=_A    )
			self.assertEqual(tensor_batch['inputs'].shape						,  (1, 3)    )
			self.assertEqual(tensor_batch['labels'].shape						,  (1,)    )







		@require_torch
		def      __lowerCAmelCase	(      self	:		Union[str, Any]    )						->				Any:
			__magic_name__ :					Optional[int]  			=  BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}    )
			__magic_name__ :					List[Any]  			=  batch.convert_to_tensors(tensor_type='pt'    )
			self.assertEqual(tensor_batch['inputs'].shape						,  (2, 3)    )
			self.assertEqual(tensor_batch['labels'].shape						,  (2,)    )
			# test converting the converted
			with CaptureStderr() as cs:
				__magic_name__ :					Union[str, Any]  			=  batch.convert_to_tensors(tensor_type='pt'    )
			self.assertFalse(len(cs.err    )						,  msg=F'should have no warning, but got {cs.err}'    )

			__magic_name__ :					Any  			=  BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}    )
			__magic_name__ :					str  			=  batch.convert_to_tensors(tensor_type='pt'						,  prepend_batch_axis=_A    )
			self.assertEqual(tensor_batch['inputs'].shape						,  (1, 3)    )
			self.assertEqual(tensor_batch['labels'].shape						,  (1,)    )







		@require_tf
		def      __lowerCAmelCase	(      self	:		Optional[Any]    )						->				List[str]:
			__magic_name__ :					str  			=  BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}    )
			__magic_name__ :					List[str]  			=  batch.convert_to_tensors(tensor_type='tf'    )
			self.assertEqual(tensor_batch['inputs'].shape						,  (2, 3)    )
			self.assertEqual(tensor_batch['labels'].shape						,  (2,)    )
			# test converting the converted
			with CaptureStderr() as cs:
				__magic_name__ :					Optional[int]  			=  batch.convert_to_tensors(tensor_type='tf'    )
			self.assertFalse(len(cs.err    )						,  msg=F'should have no warning, but got {cs.err}'    )

			__magic_name__ :					Union[str, Any]  			=  BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}    )
			__magic_name__ :					Union[str, Any]  			=  batch.convert_to_tensors(tensor_type='tf'						,  prepend_batch_axis=_A    )
			self.assertEqual(tensor_batch['inputs'].shape						,  (1, 3)    )
			self.assertEqual(tensor_batch['labels'].shape						,  (1,)    )







		@require_flax
		def      __lowerCAmelCase	(      self	:		str    )						->				Optional[Any]:
			__magic_name__ :					List[Any]  			=  BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}    )
			__magic_name__ :					int  			=  batch.convert_to_tensors(tensor_type='jax'    )
			self.assertEqual(tensor_batch['inputs'].shape						,  (2, 3)    )
			self.assertEqual(tensor_batch['labels'].shape						,  (2,)    )
			# test converting the converted
			with CaptureStderr() as cs:
				__magic_name__ :					Optional[Any]  			=  batch.convert_to_tensors(tensor_type='jax'    )
			self.assertFalse(len(cs.err    )						,  msg=F'should have no warning, but got {cs.err}'    )

			__magic_name__ :					str  			=  BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}    )
			__magic_name__ :					Dict  			=  batch.convert_to_tensors(tensor_type='jax'						,  prepend_batch_axis=_A    )
			self.assertEqual(tensor_batch['inputs'].shape						,  (1, 3)    )
			self.assertEqual(tensor_batch['labels'].shape						,  (1,)    )







		def      __lowerCAmelCase	(      self	:		str    )						->				int:
			__magic_name__ :					int  			=  [{'input_ids': np.array([0, 1, 2]    )}, {'input_ids': np.array([0, 1, 2, 3]    )}]
			__magic_name__ :					Dict  			=  BertTokenizer.from_pretrained('bert-base-cased'    )

			__magic_name__ :					Optional[int]  			=  tokenizer.pad(_A						,  padding=_A    )
			self.assertTrue(isinstance(batch['input_ids']						,  np.ndarray    )    )
			self.assertEqual(batch['input_ids'].tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
			__magic_name__ :					Dict  			=  tokenizer.pad(_A						,  padding=_A						,  return_tensors='np'    )
			self.assertTrue(isinstance(batch['input_ids']						,  np.ndarray    )    )
			self.assertEqual(batch['input_ids'].tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )







		@require_torch
		def      __lowerCAmelCase	(      self	:		List[Any]    )						->				str:
			import torch

			__magic_name__ :					List[str]  			=  [{'input_ids': torch.tensor([0, 1, 2]    )}, {'input_ids': torch.tensor([0, 1, 2, 3]    )}]
			__magic_name__ :					List[str]  			=  BertTokenizer.from_pretrained('bert-base-cased'    )

			__magic_name__ :					List[Any]  			=  tokenizer.pad(_A						,  padding=_A    )
			self.assertTrue(isinstance(batch['input_ids']						,  torch.Tensor    )    )
			self.assertEqual(batch['input_ids'].tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
			__magic_name__ :					Union[str, Any]  			=  tokenizer.pad(_A						,  padding=_A						,  return_tensors='pt'    )
			self.assertTrue(isinstance(batch['input_ids']						,  torch.Tensor    )    )
			self.assertEqual(batch['input_ids'].tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )







		@require_tf
		def      __lowerCAmelCase	(      self	:		Any    )						->				Optional[int]:
			import tensorflow as tf

			__magic_name__ :					Tuple  			=  [{'input_ids': tf.constant([0, 1, 2]    )}, {'input_ids': tf.constant([0, 1, 2, 3]    )}]
			__magic_name__ :					Any  			=  BertTokenizer.from_pretrained('bert-base-cased'    )

			__magic_name__ :					Any  			=  tokenizer.pad(_A						,  padding=_A    )
			self.assertTrue(isinstance(batch['input_ids']						,  tf.Tensor    )    )
			self.assertEqual(batch['input_ids'].numpy().tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
			__magic_name__ :					Union[str, Any]  			=  tokenizer.pad(_A						,  padding=_A						,  return_tensors='tf'    )
			self.assertTrue(isinstance(batch['input_ids']						,  tf.Tensor    )    )
			self.assertEqual(batch['input_ids'].numpy().tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )







		@require_tokenizers
		def      __lowerCAmelCase	(      self	:		Any    )						->				str:
			__magic_name__ :					str  			=  Tokenizer(WordPiece(unk_token='[UNK]'    )    )
			PreTrainedTokenizerFast(tokenizer_object=_A    )







		@require_tokenizers
		def      __lowerCAmelCase	(      self	:		Dict    )						->				Any:
			__magic_name__ :					List[Any]  			=  Tokenizer(WordPiece(unk_token='[UNK]'    )    )
			with tempfile.TemporaryDirectory() as tmpdirname:
				bert_tokenizer.save(os.path.join(_A						,  'tokenizer.json'    )    )
				PreTrainedTokenizerFast(tokenizer_file=os.path.join(_A						,  'tokenizer.json'    )    )



'''simple docstring'''

from .number_theory.prime_numbers import next_prime
class _lowerCamelCase :



		'''simple docstring'''







		def __init__(      self	:		Dict						,  _A	:		int						,  _A	:		int | None = None						,  _A	:		float | None = None						,  )						->				None:
			__magic_name__ :					Optional[int]  			=  size_table
			__magic_name__ :					int  			=  [None] * self.size_table
			__magic_name__ :					int  			=  0.75 if lim_charge is None else lim_charge
			__magic_name__ :					int  			=  1 if charge_factor is None else charge_factor
			__magic_name__ :					list  			=  []
			__magic_name__ :					dict  			=  {}







		def      __lowerCAmelCase	(      self	:		List[str]    )						->				str:
			return self._keys







		def      __lowerCAmelCase	(      self	:		Dict    )						->				Optional[Any]:
			return sum(1 for slot in self.values if slot is not None    ) / (
			    self.size_table * self.charge_factor
			)







		def      __lowerCAmelCase	(      self	:		int						,  _A	:		int    )						->				List[Any]:
			return key % self.size_table







		def      __lowerCAmelCase	(      self	:		List[Any]						,  _A	:		Tuple    )						->				Optional[int]:
			print(F'step {step_ord}'    )
			print(list(range(len(self.values    )    )    )    )
			print(self.values    )







		def      __lowerCAmelCase	(      self	:		str						,  _A	:		Optional[Any]    )						->				Optional[int]:
			__magic_name__ :					int  			=  1
			__magic_name__ :					Optional[int]  			=  values
			for value in values:
				self.insert_data(_A    )
				self._step_by_step(_A    )
				i += 1







		def      __lowerCAmelCase	(      self	:		List[Any]						,  _A	:		List[str]						,  _A	:		List[Any]    )						->				Dict:
			__magic_name__ :					str  			=  data
			__magic_name__ :					Optional[int]  			=  data







		def      __lowerCAmelCase	(      self	:		Union[str, Any]						,  _A	:		List[Any]						,  _A	:		Union[str, Any]=None    )						->				int:
			__magic_name__ :					int  			=  self.hash_function(key + 1    )

			while self.values[new_key] is not None and self.values[new_key] != key:
				if self.values.count(_A    ) > 0:
					__magic_name__ :					Dict  			=  self.hash_function(new_key + 1    )
				else:
					__magic_name__ :					Tuple  			=  None
					break

			return new_key







		def      __lowerCAmelCase	(      self	:		List[str]    )						->				Tuple:
			__magic_name__ :					Dict  			=  [value for value in self.values if value is not None]
			__magic_name__ :					int  			=  next_prime(self.size_table						,  factor=2    )
			self._keys.clear()
			__magic_name__ :					Tuple  			=  [None] * self.size_table  # hell's pointers D: don't DRY ;/
			for value in survivor_values:
				self.insert_data(_A    )







		def      __lowerCAmelCase	(      self	:		Tuple						,  _A	:		Union[str, Any]    )						->				Tuple:
			__magic_name__ :					Tuple  			=  self.hash_function(_A    )

			if self.values[key] is None:
				self._set_value(_A						,  _A    )

			elif self.values[key] == data:
				pass

			else:
				__magic_name__ :					str  			=  self._collision_resolution(_A						,  _A    )
				if collision_resolution is not None:
					self._set_value(_A						,  _A    )
				else:
					self.rehashing()
					self.insert_data(_A    )