code: 
  
  
  '''simple docstring'''
  
  import os
  import pickle
  import tempfile
  import unittest
  from typing import Callable, Optional
  
  import numpy as np
  
  from transformers import (
      BatchEncoding,
      BertTokenizer,
      BertTokenizerFast,
      PreTrainedTokenizer,
      PreTrainedTokenizerFast,
      TensorType,
      TokenSpan,
      is_tokenizers_available,
  )
  from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
  from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow
  
  
  if is_tokenizers_available():
         from tokenizers import Tokenizer
         from tokenizers.models import WordPiece
  
  
  
  
  
  
  
  class  UpperCAmelCase (       unittest.TestCase):
  
   def 		lowercase_						( self :	Tuple,					a_ :	Union[str, Any]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						list(tokenizer_class.max_model_input_sizes.keys()					)
      for model_name in sa_models[:1]:
         UpperCamelCase__ 						=						tokenizer_class.from_pretrained(a_					)
         self.assertIsNotNone(a_					)
         self.assertIsInstance(a_,					a_					)
         self.assertIsInstance(a_,					a_					)
  
         for special_tok in tokenizer.all_special_tokens:
            self.assertIsInstance(a_,					a_					)
            UpperCamelCase__ 						=						tokenizer.convert_tokens_to_ids(a_					)
            self.assertIsInstance(a_,					a_					)
  
   def 		lowercase_						( self :	Union[str, Any],					a_ :	BatchEncoding,					a_ :	Optional[Callable] = None					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						pickle.dumps(a_					)
      self.assertIsNotNone(a_					)
  
      UpperCamelCase__ 						=						pickle.loads(a_					)
  
      # Ensure is_fast is correctly restored
      self.assertEqual(be_restored.is_fast,					be_original.is_fast					)
  
      # Ensure encodings are potentially correctly restored
      if be_original.is_fast:
         self.assertIsNotNone(be_restored.encodings					)
      else:
         self.assertIsNone(be_restored.encodings					)
  
      # Ensure the keys are the same
      for original_v, restored_v in zip(be_original.values(),					be_restored.values()					):
         if equal_op:
            self.assertTrue(equal_op(a_,					a_					)					)
         else:
            self.assertEqual(a_,					a_					)
  
   @slow
   def 		lowercase_						( self :	Optional[int]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      self.check_tokenizer_from_pretrained(a_					)
  
   def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      self.assertEqual(TensorType("tf"					),					TensorType.TENSORFLOW					)
      self.assertEqual(TensorType("pt"					),					TensorType.PYTORCH					)
      self.assertEqual(TensorType("np"					),					TensorType.NUMPY					)
  
   @require_tokenizers
   def 		lowercase_						( self :	List[Any]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      import numpy as np
  
      UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
      UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
  
      # Python no tensor
      with self.subTest("BatchEncoding (Python, return_tensors=None)"					):
         self.assert_dump_and_restore(tokenizer_p("Small example to encode"					)					)
  
      with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"					):
         self.assert_dump_and_restore(
             tokenizer_p("Small example to encode",					return_tensors=TensorType.NUMPY					),					np.array_equal					)
  
      with self.subTest("BatchEncoding (Rust, return_tensors=None)"					):
         self.assert_dump_and_restore(tokenizer_r("Small example to encode"					)					)
  
      with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"					):
         self.assert_dump_and_restore(
             tokenizer_r("Small example to encode",					return_tensors=TensorType.NUMPY					),					np.array_equal					)
  
   @require_tf
   @require_tokenizers
   def 		lowercase_						( self :	str					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      import tensorflow as tf
  
      def tf_array_equals(a_ :	Tuple,					a_ :	str					):
         return tf.reduce_all(tf.equal(a_,					a_					)					)
  
      UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
      UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
  
      with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"					):
         self.assert_dump_and_restore(
             tokenizer_p("Small example to encode",					return_tensors=TensorType.TENSORFLOW					),					a_					)
  
      with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"					):
         self.assert_dump_and_restore(
             tokenizer_r("Small example to encode",					return_tensors=TensorType.TENSORFLOW					),					a_					)
  
   @require_torch
   @require_tokenizers
   def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      import torch
  
      UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
      UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
  
      with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"					):
         self.assert_dump_and_restore(
             tokenizer_p("Small example to encode",					return_tensors=TensorType.PYTORCH					),					torch.equal					)
  
      with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"					):
         self.assert_dump_and_restore(
             tokenizer_r("Small example to encode",					return_tensors=TensorType.PYTORCH					),					torch.equal					)
  
   @require_tokenizers
   def 		lowercase_						( self :	Optional[int]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
      UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
  
      with self.subTest("Python Tokenizer"					):
         self.assertFalse(tokenizer_p("Small example to_encode"					).is_fast					)
  
      with self.subTest("Rust Tokenizer"					):
         self.assertTrue(tokenizer_r("Small example to_encode"					).is_fast					)
  
   @require_tokenizers
   def 		lowercase_						( self :	str					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
      UpperCamelCase__ 						=						tokenizer_r(["Test", "\xad", "test"],					is_split_into_words=a_					)
  
      self.assertEqual(encoded.word_to_tokens(0					),					TokenSpan(start=1,					end=2					)					)
      self.assertEqual(encoded.word_to_tokens(1					),					a_					)
      self.assertEqual(encoded.word_to_tokens(2					),					TokenSpan(start=2,					end=3					)					)
  
   def 		lowercase_						( self :	Optional[int]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
      UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="np"					)
      self.assertEqual(tensor_batch["inputs"].shape,					(2, 3)					)
      self.assertEqual(tensor_batch["labels"].shape,					(2,)					)
      # test converting the converted
      with CaptureStderr() as cs:
         UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="np"					)
      self.assertFalse(len(cs.err					),					msg=f'should have no warning, but got {cs.err}'					)
  
      UpperCamelCase__ 						=						BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
      UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="np",					prepend_batch_axis=a_					)
      self.assertEqual(tensor_batch["inputs"].shape,					(1, 3)					)
      self.assertEqual(tensor_batch["labels"].shape,					(1,)					)
  
   @require_torch
   def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
      UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="pt"					)
      self.assertEqual(tensor_batch["inputs"].shape,					(2, 3)					)
      self.assertEqual(tensor_batch["labels"].shape,					(2,)					)
      # test converting the converted
      with CaptureStderr() as cs:
         UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="pt"					)
      self.assertFalse(len(cs.err					),					msg=f'should have no warning, but got {cs.err}'					)
  
      UpperCamelCase__ 						=						BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
      UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="pt",					prepend_batch_axis=a_					)
      self.assertEqual(tensor_batch["inputs"].shape,					(1, 3)					)
      self.assertEqual(tensor_batch["labels"].shape,					(1,)					)
  
   @require_tf
   def 		lowercase_						( self :	Optional[Any]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
      UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="tf"					)
      self.assertEqual(tensor_batch["inputs"].shape,					(2, 3)					)
      self.assertEqual(tensor_batch["labels"].shape,					(2,)					)
      # test converting the converted
      with CaptureStderr() as cs:
         UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="tf"					)
      self.assertFalse(len(cs.err					),					msg=f'should have no warning, but got {cs.err}'					)
  
      UpperCamelCase__ 						=						BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
      UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="tf",					prepend_batch_axis=a_					)
      self.assertEqual(tensor_batch["inputs"].shape,					(1, 3)					)
      self.assertEqual(tensor_batch["labels"].shape,					(1,)					)
  
   @require_flax
   def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
      UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="jax"					)
      self.assertEqual(tensor_batch["inputs"].shape,					(2, 3)					)
      self.assertEqual(tensor_batch["labels"].shape,					(2,)					)
      # test converting the converted
      with CaptureStderr() as cs:
         UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="jax"					)
      self.assertFalse(len(cs.err					),					msg=f'should have no warning, but got {cs.err}'					)
  
      UpperCamelCase__ 						=						BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
      UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="jax",					prepend_batch_axis=a_					)
      self.assertEqual(tensor_batch["inputs"].shape,					(1, 3)					)
      self.assertEqual(tensor_batch["labels"].shape,					(1,)					)
  
   def 		lowercase_						( self :	List[Any]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						[{"input_ids": np.array([0, 1, 2]					)}, {"input_ids": np.array([0, 1, 2, 3]					)}]
      UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  
      UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_					)
      self.assertTrue(isinstance(batch["input_ids"],					np.ndarray					)					)
      self.assertEqual(batch["input_ids"].tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
      UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_,					return_tensors="np"					)
      self.assertTrue(isinstance(batch["input_ids"],					np.ndarray					)					)
      self.assertEqual(batch["input_ids"].tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
  
   @require_torch
   def 		lowercase_						( self :	List[Any]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      import torch
  
      UpperCamelCase__ 						=						[{"input_ids": torch.tensor([0, 1, 2]					)}, {"input_ids": torch.tensor([0, 1, 2, 3]					)}]
      UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  
      UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_					)
      self.assertTrue(isinstance(batch["input_ids"],					torch.Tensor					)					)
      self.assertEqual(batch["input_ids"].tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
      UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_,					return_tensors="pt"					)
      self.assertTrue(isinstance(batch["input_ids"],					torch.Tensor					)					)
      self.assertEqual(batch["input_ids"].tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
  
   @require_tf
   def 		lowercase_						( self :	int					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      import tensorflow as tf
  
      UpperCamelCase__ 						=						[{"input_ids": tf.constant([0, 1, 2]					)}, {"input_ids": tf.constant([0, 1, 2, 3]					)}]
      UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  
      UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_					)
      self.assertTrue(isinstance(batch["input_ids"],					tf.Tensor					)					)
      self.assertEqual(batch["input_ids"].numpy().tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
      UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_,					return_tensors="tf"					)
      self.assertTrue(isinstance(batch["input_ids"],					tf.Tensor					)					)
      self.assertEqual(batch["input_ids"].numpy().tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
  
   @require_tokenizers
   def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						Tokenizer(WordPiece(unk_token="[UNK]"					)					)
      PreTrainedTokenizerFast(tokenizer_object=a_					)
  
   @require_tokenizers
   def 		lowercase_						( self :	Optional[int]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						Tokenizer(WordPiece(unk_token="[UNK]"					)					)
      with tempfile.TemporaryDirectory() as tmpdirname:
         bert_tokenizer.save(os.path.join(a_,					"tokenizer.json"					)					)
         PreTrainedTokenizerFast(tokenizer_file=os.path.join(a_,					"tokenizer.json"					)					)
style_context: 
  
  
  '''simple docstring'''
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_sentencepiece_available,
      is_tokenizers_available,
      is_torch_available,
  )
  
  
  __lowercase:		int							=     {
      "configuration_open_llama": ["OPEN_LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP", "OpenLlamaConfig"],
  }
  
  try:
         if not is_sentencepiece_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         __lowercase:		str							=     ["LlamaTokenizer"]
  
  try:
         if not is_tokenizers_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         __lowercase:		List[Any]							=     ["LlamaTokenizerFast"]
  
  try:
         if not is_torch_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         __lowercase:		Union[str, Any]							=     [
             "OpenLlamaForCausalLM",
             "OpenLlamaModel",
             "OpenLlamaPreTrainedModel",
             "OpenLlamaForSequenceClassification",
         ]
  
  
  if TYPE_CHECKING:
         from .configuration_open_llama import OPEN_LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenLlamaConfig
  
         try:
                if not is_sentencepiece_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from transformers import LlamaTokenizer
  
         try:
                if not is_tokenizers_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from transformers import LlamaTokenizerFast
  
         try:
                if not is_torch_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from .modeling_open_llama import (
                    OpenLlamaForCausalLM,
                    OpenLlamaForSequenceClassification,
                    OpenLlamaModel,
                    OpenLlamaPreTrainedModel,
                )
  
  
  else:
         import sys
  
         __lowercase:		Tuple							=     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)