



'''simple docstring'''






import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
   import tensorflow as tf

if is_tensorflow_text_available():
   from transformers.models.bert import TFBertTokenizer


A_							: int						 =	["""bert-base-uncased""", """bert-base-cased"""]
A_							: Dict						 =	"""hf-internal-testing/tiny-bert-tf-only"""

if is_tf_available():



   class 							lowercase       (  tf.keras.Model				):


    """simple docstring"""


    def __init__(							self  ,a_							)		->   Optional[Any]:
           super().__init__()
           _UpperCAmelCase  :						Dict			     = tokenizer
           _UpperCAmelCase  :						Optional[int]			     = AutoConfig.from_pretrained(a_							)
           _UpperCAmelCase  :						Any			     = TFAutoModel.from_config(a_							)







    def _snake_case (							self  ,a_							)		->   int:
           _UpperCAmelCase  :						List[Any]			     = self.tokenizer(a_							)
           _UpperCAmelCase  :						List[Any]			     = self.bert(**a_							)
           return out["pooler_output"]



@require_tf
@require_tensorflow_text
class 							lowercase       (  unittest.TestCase				):


 """simple docstring"""


 def _snake_case (							self							)		->   Optional[Any]:
        super().setUp()

        _UpperCAmelCase  :						Optional[Any]			     = [
            BertTokenizer.from_pretrained(a_							) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
        ]  # repeat for when fast_bert_tokenizer=false
        _UpperCAmelCase  :						str			     = [TFBertTokenizer.from_pretrained(a_							) for checkpoint in TOKENIZER_CHECKPOINTS] + [
            TFBertTokenizer.from_pretrained(a_  ,use_fast_bert_tokenizer=a_							)
            for checkpoint in TOKENIZER_CHECKPOINTS
        ]
        assert len(self.tokenizers							) == len(self.tf_tokenizers							)

        _UpperCAmelCase  :						List[Any]			     = [
            """This is a straightforward English test sentence.""",
            """This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.""",
            """Now we're going to add some Chinese: 一 二 三 一二三""",
            """And some much more rare Chinese: 齉 堃 齉堃""",
            """Je vais aussi écrire en français pour tester les accents""",
            """Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ""",
        ]
        _UpperCAmelCase  :						List[Any]			     = list(zip(self.test_sentences  ,self.test_sentences[::-1]							)							)







 def _snake_case (							self							)		->   str:
        for tokenizer, tf_tokenizer in zip(self.tokenizers  ,self.tf_tokenizers							):
               for test_inputs in (self.test_sentences, self.paired_sentences):
                      _UpperCAmelCase  :						Dict			     = tokenizer(a_  ,return_tensors="""tf"""  ,padding="""longest"""							)
                      _UpperCAmelCase  :						int			     = tf_tokenizer(a_							)

                      for key in python_outputs.keys():
                             self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape							)							)
                             self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]  ,tf.intaa							) == tf_outputs[key]							)							)







 @slow
 def _snake_case (							self							)		->   Any:
        for tf_tokenizer in self.tf_tokenizers:
               _UpperCAmelCase  :						str			     = tf_tokenizer(self.paired_sentences							)
               _UpperCAmelCase  :						Optional[Any]			     = tf_tokenizer(
                   text=[sentence[0] for sentence in self.paired_sentences]  ,text_pair=[sentence[1] for sentence in self.paired_sentences]  ,)
               for key in merged_outputs.keys():
                      self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]  ,tf.intaa							) == separated_outputs[key]							)							)







 @slow
 def _snake_case (							self							)		->   Optional[Any]:
        for tf_tokenizer in self.tf_tokenizers:
               _UpperCAmelCase  :						Optional[Any]			     = tf.function(a_							)
               for test_inputs in (self.test_sentences, self.paired_sentences):
                      _UpperCAmelCase  :						Optional[int]			     = tf.constant(a_							)
                      _UpperCAmelCase  :						int			     = compiled_tokenizer(a_							)
                      _UpperCAmelCase  :						int			     = tf_tokenizer(a_							)

                      for key in eager_outputs.keys():
                             self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]							)							)







 @slow
 def _snake_case (							self							)		->   Optional[int]:
        for tf_tokenizer in self.tf_tokenizers:
               _UpperCAmelCase  :						List[Any]			     = ModelToSave(tokenizer=a_							)
               _UpperCAmelCase  :						str			     = tf.convert_to_tensor(self.test_sentences							)
               _UpperCAmelCase  :						List[Any]			     = model(a_							)  # Build model with some sample inputs
               with TemporaryDirectory() as tempdir:
                      _UpperCAmelCase  :						List[str]			     = Path(a_							) / """saved.model"""
                      model.save(a_							)
                      _UpperCAmelCase  :						Tuple			     = tf.keras.models.load_model(a_							)
               _UpperCAmelCase  :						List[str]			     = loaded_model(a_							)
               # We may see small differences because the loaded model is compiled, so we need an epsilon for the test
               self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output							)							)  ,1E-5							)








'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


A_							: Optional[Any]						 =	{
    """configuration_llama""": ["""LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LlamaConfig"""],
}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							: List[str]						 =	["""LlamaTokenizer"""]

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							: Dict						 =	["""LlamaTokenizerFast"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							: Optional[int]						 =	[
       """LlamaForCausalLM""",
       """LlamaModel""",
       """LlamaPreTrainedModel""",
       """LlamaForSequenceClassification""",
   ]


if TYPE_CHECKING:
   from .configuration_llama import LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, LlamaConfig

   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_llama import LlamaTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_llama_fast import LlamaTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_llama import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaPreTrainedModel


else:
   import sys

   A_							: Any						 =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



