





"""simple docstring"""


import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
		from tokenizers import Tokenizer
		from tokenizers.models import WordPiece






class        lowercase__ (      unittest.TestCase  ):

			"""simple docstring"""







			def      _a			(			self	,      _A						):






									'''simple docstring'''







									UpperCamelCase							:		List[str]							      = list(tokenizer_class.max_model_input_sizes.keys()						)
									for model_name in sa_models[:1]:
															UpperCamelCase							:		Dict							      = tokenizer_class.from_pretrained(_A						)
															self.assertIsNotNone(_A						)
															self.assertIsInstance(_A	,      _A						)
															self.assertIsInstance(_A	,      _A						)

															for special_tok in tokenizer.all_special_tokens:
																					self.assertIsInstance(_A	,      _A						)
																					UpperCamelCase							:		Dict							      = tokenizer.convert_tokens_to_ids(_A						)
																					self.assertIsInstance(_A	,      _A						)







			def      _a			(			self	,      _A	,      _A = None						):






									'''simple docstring'''







									UpperCamelCase							:		Tuple							      = pickle.dumps(_A						)
									self.assertIsNotNone(_A						)

									UpperCamelCase							:		Tuple							      = pickle.loads(_A						)

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast	,      be_original.is_fast						)

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
															self.assertIsNotNone(be_restored.encodings						)
									else:
															self.assertIsNone(be_restored.encodings						)

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()	,      be_restored.values()						):
															if equal_op:
																					self.assertTrue(equal_op(_A	,      _A						)						)
															else:
																					self.assertEqual(_A	,      _A						)







			@slow
			def      _a			(			self						):






									'''simple docstring'''







									self.check_tokenizer_from_pretrained(_A						)







			def      _a			(			self						):






									'''simple docstring'''







									self.assertEqual(TensorType("""tf"""						)	,      TensorType.TENSORFLOW						)
									self.assertEqual(TensorType("""pt"""						)	,      TensorType.PYTORCH						)
									self.assertEqual(TensorType("""np"""						)	,      TensorType.NUMPY						)







			@require_tokenizers
			def      _a			(			self						):






									'''simple docstring'''







									import numpy as np

									UpperCamelCase							:		Any							      = BertTokenizer.from_pretrained("""bert-base-cased"""						)
									UpperCamelCase							:		Tuple							      = BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									# Python no tensor
									with self.subTest("""BatchEncoding (Python, return_tensors=None)"""						):
															self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""						):
															self.assert_dump_and_restore(
															    tokenizer_p("""Small example to encode"""	,      return_tensors=TensorType.NUMPY						)	,      np.array_equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""						):
															self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""						):
															self.assert_dump_and_restore(
															    tokenizer_r("""Small example to encode"""	,      return_tensors=TensorType.NUMPY						)	,      np.array_equal						)







			@require_tf
			@require_tokenizers
			def      _a			(			self						):






									'''simple docstring'''







									import tensorflow as tf

									def tf_array_equals(_A	,      _A						):
															return tf.reduce_all(tf.equal(_A	,      _A						)						)

									UpperCamelCase							:		int							      = BertTokenizer.from_pretrained("""bert-base-cased"""						)
									UpperCamelCase							:		Any							      = BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""						):
															self.assert_dump_and_restore(
															    tokenizer_p("""Small example to encode"""	,      return_tensors=TensorType.TENSORFLOW						)	,      _A						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""						):
															self.assert_dump_and_restore(
															    tokenizer_r("""Small example to encode"""	,      return_tensors=TensorType.TENSORFLOW						)	,      _A						)







			@require_torch
			@require_tokenizers
			def      _a			(			self						):






									'''simple docstring'''







									import torch

									UpperCamelCase							:		Optional[int]							      = BertTokenizer.from_pretrained("""bert-base-cased"""						)
									UpperCamelCase							:		str							      = BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""						):
															self.assert_dump_and_restore(
															    tokenizer_p("""Small example to encode"""	,      return_tensors=TensorType.PYTORCH						)	,      torch.equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""						):
															self.assert_dump_and_restore(
															    tokenizer_r("""Small example to encode"""	,      return_tensors=TensorType.PYTORCH						)	,      torch.equal						)







			@require_tokenizers
			def      _a			(			self						):






									'''simple docstring'''







									UpperCamelCase							:		List[str]							      = BertTokenizer.from_pretrained("""bert-base-cased"""						)
									UpperCamelCase							:		List[str]							      = BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""Python Tokenizer"""						):
															self.assertFalse(tokenizer_p("""Small example to_encode"""						).is_fast						)

									with self.subTest("""Rust Tokenizer"""						):
															self.assertTrue(tokenizer_r("""Small example to_encode"""						).is_fast						)







			@require_tokenizers
			def      _a			(			self						):






									'''simple docstring'''







									UpperCamelCase							:		Tuple							      = BertTokenizerFast.from_pretrained("""bert-base-cased"""						)
									UpperCamelCase							:		Optional[int]							      = tokenizer_r(["""Test""", """\xad""", """test"""]	,      is_split_into_words=_A						)

									self.assertEqual(encoded.word_to_tokens(0						)	,      TokenSpan(start=1	,      end=2						)						)
									self.assertEqual(encoded.word_to_tokens(1						)	,      _A						)
									self.assertEqual(encoded.word_to_tokens(2						)	,      TokenSpan(start=2	,      end=3						)						)







			def      _a			(			self						):






									'''simple docstring'''







									UpperCamelCase							:		List[str]							      = BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									UpperCamelCase							:		List[Any]							      = batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape	,      (2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape	,      (2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
															UpperCamelCase							:		Optional[int]							      = batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertFalse(len(cs.err						)	,      msg=f"""should have no warning, but got {cs.err}"""						)

									UpperCamelCase							:		Any							      = BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									UpperCamelCase							:		Union[str, Any]							      = batch.convert_to_tensors(tensor_type="""np"""	,      prepend_batch_axis=_A						)
									self.assertEqual(tensor_batch["""inputs"""].shape	,      (1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape	,      (1,)						)







			@require_torch
			def      _a			(			self						):






									'''simple docstring'''







									UpperCamelCase							:		List[Any]							      = BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									UpperCamelCase							:		Dict							      = batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape	,      (2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape	,      (2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
															UpperCamelCase							:		int							      = batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertFalse(len(cs.err						)	,      msg=f"""should have no warning, but got {cs.err}"""						)

									UpperCamelCase							:		Dict							      = BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									UpperCamelCase							:		List[str]							      = batch.convert_to_tensors(tensor_type="""pt"""	,      prepend_batch_axis=_A						)
									self.assertEqual(tensor_batch["""inputs"""].shape	,      (1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape	,      (1,)						)







			@require_tf
			def      _a			(			self						):






									'''simple docstring'''







									UpperCamelCase							:		Optional[Any]							      = BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									UpperCamelCase							:		Any							      = batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape	,      (2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape	,      (2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
															UpperCamelCase							:		Optional[Any]							      = batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertFalse(len(cs.err						)	,      msg=f"""should have no warning, but got {cs.err}"""						)

									UpperCamelCase							:		str							      = BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									UpperCamelCase							:		str							      = batch.convert_to_tensors(tensor_type="""tf"""	,      prepend_batch_axis=_A						)
									self.assertEqual(tensor_batch["""inputs"""].shape	,      (1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape	,      (1,)						)







			@require_flax
			def      _a			(			self						):






									'''simple docstring'''







									UpperCamelCase							:		Dict							      = BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									UpperCamelCase							:		int							      = batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape	,      (2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape	,      (2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
															UpperCamelCase							:		Union[str, Any]							      = batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertFalse(len(cs.err						)	,      msg=f"""should have no warning, but got {cs.err}"""						)

									UpperCamelCase							:		List[Any]							      = BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									UpperCamelCase							:		str							      = batch.convert_to_tensors(tensor_type="""jax"""	,      prepend_batch_axis=_A						)
									self.assertEqual(tensor_batch["""inputs"""].shape	,      (1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape	,      (1,)						)







			def      _a			(			self						):






									'''simple docstring'''







									UpperCamelCase							:		List[Any]							      = [{"""input_ids""": np.array([0, 1, 2]						)}, {"""input_ids""": np.array([0, 1, 2, 3]						)}]
									UpperCamelCase							:		List[str]							      = BertTokenizer.from_pretrained("""bert-base-cased"""						)

									UpperCamelCase							:		str							      = tokenizer.pad(_A	,      padding=_A						)
									self.assertTrue(isinstance(batch["""input_ids"""]	,      np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									UpperCamelCase							:		int							      = tokenizer.pad(_A	,      padding=_A	,      return_tensors="""np"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]	,      np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)







			@require_torch
			def      _a			(			self						):






									'''simple docstring'''







									import torch

									UpperCamelCase							:		str							      = [{"""input_ids""": torch.tensor([0, 1, 2]						)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]						)}]
									UpperCamelCase							:		List[Any]							      = BertTokenizer.from_pretrained("""bert-base-cased"""						)

									UpperCamelCase							:		Optional[int]							      = tokenizer.pad(_A	,      padding=_A						)
									self.assertTrue(isinstance(batch["""input_ids"""]	,      torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									UpperCamelCase							:		Optional[Any]							      = tokenizer.pad(_A	,      padding=_A	,      return_tensors="""pt"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]	,      torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)







			@require_tf
			def      _a			(			self						):






									'''simple docstring'''







									import tensorflow as tf

									UpperCamelCase							:		List[str]							      = [{"""input_ids""": tf.constant([0, 1, 2]						)}, {"""input_ids""": tf.constant([0, 1, 2, 3]						)}]
									UpperCamelCase							:		List[Any]							      = BertTokenizer.from_pretrained("""bert-base-cased"""						)

									UpperCamelCase							:		Any							      = tokenizer.pad(_A	,      padding=_A						)
									self.assertTrue(isinstance(batch["""input_ids"""]	,      tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									UpperCamelCase							:		Tuple							      = tokenizer.pad(_A	,      padding=_A	,      return_tensors="""tf"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]	,      tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()	,      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)







			@require_tokenizers
			def      _a			(			self						):






									'''simple docstring'''







									UpperCamelCase							:		Optional[int]							      = Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									PreTrainedTokenizerFast(tokenizer_object=_A						)







			@require_tokenizers
			def      _a			(			self						):






									'''simple docstring'''







									UpperCamelCase							:		Tuple							      = Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									with tempfile.TemporaryDirectory() as tmpdirname:
															bert_tokenizer.save(os.path.join(_A	,      """tokenizer.json"""						)						)
															PreTrainedTokenizerFast(tokenizer_file=os.path.join(_A	,      """tokenizer.json"""						)						)










"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


__magic_name__					:				Any	= logging.get_logger(__name__)

__magic_name__					:				Dict	= {
    """uw-madison/yoso-4096""": """https://huggingface.co/uw-madison/yoso-4096/resolve/main/config.json""",
    # See all YOSO models at https://huggingface.co/models?filter=yoso
}






class        lowercase__ (      __SCREAMING_SNAKE_CASE  ):

			"""simple docstring"""
			__lowerCAmelCase    :							Optional[int]								=     """yoso"""







			def __init__(			self	,      _A=5_0_2_6_5	,      _A=7_6_8	,      _A=1_2	,      _A=1_2	,      _A=3_0_7_2	,      _A="gelu"	,      _A=0.1	,      _A=0.1	,      _A=4_0_9_6	,      _A=1	,      _A=0.02	,      _A=1e-1_2	,      _A="absolute"	,      _A=True	,      _A=9	,      _A=6_4	,      _A=None	,      _A=True	,      _A=True	,      _A=1	,      _A=0	,      _A=2	,      **_A	,      ):






									'''simple docstring'''







									super().__init__(pad_token_id=_A	,      bos_token_id=_A	,      eos_token_id=_A	,      **_A						)

									UpperCamelCase							:		Any							      = vocab_size
									UpperCamelCase							:		Any							      = max_position_embeddings
									UpperCamelCase							:		int							      = hidden_size
									UpperCamelCase							:		List[Any]							      = num_hidden_layers
									UpperCamelCase							:		Any							      = num_attention_heads
									UpperCamelCase							:		Optional[Any]							      = intermediate_size
									UpperCamelCase							:		Optional[int]							      = hidden_act
									UpperCamelCase							:		List[Any]							      = hidden_dropout_prob
									UpperCamelCase							:		int							      = attention_probs_dropout_prob
									UpperCamelCase							:		Any							      = initializer_range
									UpperCamelCase							:		Optional[Any]							      = type_vocab_size
									UpperCamelCase							:		Any							      = layer_norm_eps
									UpperCamelCase							:		List[Any]							      = position_embedding_type
									UpperCamelCase							:		Optional[Any]							      = use_expectation
									UpperCamelCase							:		Any							      = hash_code_len
									UpperCamelCase							:		Optional[int]							      = num_hash
									UpperCamelCase							:		List[Any]							      = conv_window
									UpperCamelCase							:		Tuple							      = use_fast_hash
									UpperCamelCase							:		Dict							      = lsh_backward



