



'''simple docstring'''






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
     from tokenizers import Tokenizer
     from tokenizers.models import WordPiece



class 							lowercase (						unittest.TestCase       ):


 """simple docstring"""


 def _snake_case				(							self						,a_							)	->     Any:
        _UpperCAmelCase						:		str      =				list(tokenizer_class.max_model_input_sizes.keys()							)
        for model_name in sa_models[:1]:
               _UpperCAmelCase						:		Optional[int]      =				tokenizer_class.from_pretrained(_lowerCamelCase							)
               self.assertIsNotNone(_lowerCamelCase							)
               self.assertIsInstance(_lowerCamelCase						,_lowerCamelCase							)
               self.assertIsInstance(_lowerCamelCase						,_lowerCamelCase							)

               for special_tok in tokenizer.all_special_tokens:
                      self.assertIsInstance(_lowerCamelCase						,_lowerCamelCase							)
                      _UpperCAmelCase						:		Any      =				tokenizer.convert_tokens_to_ids(_lowerCamelCase							)
                      self.assertIsInstance(_lowerCamelCase						,_lowerCamelCase							)







 def _snake_case				(							self						,a_						,a_ = None							)	->     int:
        _UpperCAmelCase						:		Optional[Any]      =				pickle.dumps(_lowerCamelCase							)
        self.assertIsNotNone(_lowerCamelCase							)

        _UpperCAmelCase						:		str      =				pickle.loads(_lowerCamelCase							)

        # Ensure is_fast is correctly restored
        self.assertEqual(be_restored.is_fast						,be_original.is_fast							)

        # Ensure encodings are potentially correctly restored
        if be_original.is_fast:
               self.assertIsNotNone(be_restored.encodings							)
        else:
               self.assertIsNone(be_restored.encodings							)

        # Ensure the keys are the same
        for original_v, restored_v in zip(be_original.values()						,be_restored.values()							):
               if equal_op:
                      self.assertTrue(equal_op(_lowerCamelCase						,_lowerCamelCase							)							)
               else:
                      self.assertEqual(_lowerCamelCase						,_lowerCamelCase							)







 @slow
 def _snake_case				(							self							)	->     Tuple:
        self.check_tokenizer_from_pretrained(_lowerCamelCase							)







 def _snake_case				(							self							)	->     Tuple:
        self.assertEqual(TensorType("""tf"""							)						,TensorType.TENSORFLOW							)
        self.assertEqual(TensorType("""pt"""							)						,TensorType.PYTORCH							)
        self.assertEqual(TensorType("""np"""							)						,TensorType.NUMPY							)







 @require_tokenizers
 def _snake_case				(							self							)	->     str:
        import numpy as np

        _UpperCAmelCase						:		List[str]      =				BertTokenizer.from_pretrained("""bert-base-cased"""							)
        _UpperCAmelCase						:		List[str]      =				BertTokenizerFast.from_pretrained("""bert-base-cased"""							)

        # Python no tensor
        with self.subTest("""BatchEncoding (Python, return_tensors=None)"""							):
               self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""							)							)

        with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""							):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""						,return_tensors=TensorType.NUMPY							)						,np.array_equal							)

        with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""							):
               self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""							)							)

        with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""							):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""						,return_tensors=TensorType.NUMPY							)						,np.array_equal							)







 @require_tf
 @require_tokenizers
 def _snake_case				(							self							)	->     Optional[int]:
        import tensorflow as tf

        def tf_array_equals(a_						,a_							):
               return tf.reduce_all(tf.equal(_lowerCamelCase						,_lowerCamelCase							)							)

        _UpperCAmelCase						:		Union[str, Any]      =				BertTokenizer.from_pretrained("""bert-base-cased"""							)
        _UpperCAmelCase						:		str      =				BertTokenizerFast.from_pretrained("""bert-base-cased"""							)

        with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""							):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""						,return_tensors=TensorType.TENSORFLOW							)						,_lowerCamelCase							)

        with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""							):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""						,return_tensors=TensorType.TENSORFLOW							)						,_lowerCamelCase							)







 @require_torch
 @require_tokenizers
 def _snake_case				(							self							)	->     Any:
        import torch

        _UpperCAmelCase						:		Any      =				BertTokenizer.from_pretrained("""bert-base-cased"""							)
        _UpperCAmelCase						:		Optional[int]      =				BertTokenizerFast.from_pretrained("""bert-base-cased"""							)

        with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""							):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""						,return_tensors=TensorType.PYTORCH							)						,torch.equal							)

        with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""							):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""						,return_tensors=TensorType.PYTORCH							)						,torch.equal							)







 @require_tokenizers
 def _snake_case				(							self							)	->     Dict:
        _UpperCAmelCase						:		List[str]      =				BertTokenizer.from_pretrained("""bert-base-cased"""							)
        _UpperCAmelCase						:		str      =				BertTokenizerFast.from_pretrained("""bert-base-cased"""							)

        with self.subTest("""Python Tokenizer"""							):
               self.assertFalse(tokenizer_p("""Small example to_encode"""							).is_fast							)

        with self.subTest("""Rust Tokenizer"""							):
               self.assertTrue(tokenizer_r("""Small example to_encode"""							).is_fast							)







 @require_tokenizers
 def _snake_case				(							self							)	->     Dict:
        _UpperCAmelCase						:		Optional[int]      =				BertTokenizerFast.from_pretrained("""bert-base-cased"""							)
        _UpperCAmelCase						:		Union[str, Any]      =				tokenizer_r(["""Test""", """\xad""", """test"""]						,is_split_into_words=_lowerCamelCase							)

        self.assertEqual(encoded.word_to_tokens(0							)						,TokenSpan(start=1						,end=2							)							)
        self.assertEqual(encoded.word_to_tokens(1							)						,_lowerCamelCase							)
        self.assertEqual(encoded.word_to_tokens(2							)						,TokenSpan(start=2						,end=3							)							)







 def _snake_case				(							self							)	->     Optional[int]:
        _UpperCAmelCase						:		Tuple      =				BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}							)
        _UpperCAmelCase						:		Any      =				batch.convert_to_tensors(tensor_type="""np"""							)
        self.assertEqual(tensor_batch["""inputs"""].shape						,(2, 3)							)
        self.assertEqual(tensor_batch["""labels"""].shape						,(2,)							)
        # test converting the converted
        with CaptureStderr() as cs:
               _UpperCAmelCase						:		Any      =				batch.convert_to_tensors(tensor_type="""np"""							)
        self.assertFalse(len(cs.err							)						,msg=f'''should have no warning, but got {cs.err}'''							)

        _UpperCAmelCase						:		Any      =				BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}							)
        _UpperCAmelCase						:		List[Any]      =				batch.convert_to_tensors(tensor_type="""np"""						,prepend_batch_axis=_lowerCamelCase							)
        self.assertEqual(tensor_batch["""inputs"""].shape						,(1, 3)							)
        self.assertEqual(tensor_batch["""labels"""].shape						,(1,)							)







 @require_torch
 def _snake_case				(							self							)	->     List[Any]:
        _UpperCAmelCase						:		Union[str, Any]      =				BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}							)
        _UpperCAmelCase						:		Optional[Any]      =				batch.convert_to_tensors(tensor_type="""pt"""							)
        self.assertEqual(tensor_batch["""inputs"""].shape						,(2, 3)							)
        self.assertEqual(tensor_batch["""labels"""].shape						,(2,)							)
        # test converting the converted
        with CaptureStderr() as cs:
               _UpperCAmelCase						:		Union[str, Any]      =				batch.convert_to_tensors(tensor_type="""pt"""							)
        self.assertFalse(len(cs.err							)						,msg=f'''should have no warning, but got {cs.err}'''							)

        _UpperCAmelCase						:		int      =				BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}							)
        _UpperCAmelCase						:		Union[str, Any]      =				batch.convert_to_tensors(tensor_type="""pt"""						,prepend_batch_axis=_lowerCamelCase							)
        self.assertEqual(tensor_batch["""inputs"""].shape						,(1, 3)							)
        self.assertEqual(tensor_batch["""labels"""].shape						,(1,)							)







 @require_tf
 def _snake_case				(							self							)	->     Dict:
        _UpperCAmelCase						:		Any      =				BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}							)
        _UpperCAmelCase						:		Optional[Any]      =				batch.convert_to_tensors(tensor_type="""tf"""							)
        self.assertEqual(tensor_batch["""inputs"""].shape						,(2, 3)							)
        self.assertEqual(tensor_batch["""labels"""].shape						,(2,)							)
        # test converting the converted
        with CaptureStderr() as cs:
               _UpperCAmelCase						:		List[Any]      =				batch.convert_to_tensors(tensor_type="""tf"""							)
        self.assertFalse(len(cs.err							)						,msg=f'''should have no warning, but got {cs.err}'''							)

        _UpperCAmelCase						:		List[str]      =				BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}							)
        _UpperCAmelCase						:		Optional[int]      =				batch.convert_to_tensors(tensor_type="""tf"""						,prepend_batch_axis=_lowerCamelCase							)
        self.assertEqual(tensor_batch["""inputs"""].shape						,(1, 3)							)
        self.assertEqual(tensor_batch["""labels"""].shape						,(1,)							)







 @require_flax
 def _snake_case				(							self							)	->     Dict:
        _UpperCAmelCase						:		str      =				BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}							)
        _UpperCAmelCase						:		int      =				batch.convert_to_tensors(tensor_type="""jax"""							)
        self.assertEqual(tensor_batch["""inputs"""].shape						,(2, 3)							)
        self.assertEqual(tensor_batch["""labels"""].shape						,(2,)							)
        # test converting the converted
        with CaptureStderr() as cs:
               _UpperCAmelCase						:		Any      =				batch.convert_to_tensors(tensor_type="""jax"""							)
        self.assertFalse(len(cs.err							)						,msg=f'''should have no warning, but got {cs.err}'''							)

        _UpperCAmelCase						:		Optional[Any]      =				BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}							)
        _UpperCAmelCase						:		List[str]      =				batch.convert_to_tensors(tensor_type="""jax"""						,prepend_batch_axis=_lowerCamelCase							)
        self.assertEqual(tensor_batch["""inputs"""].shape						,(1, 3)							)
        self.assertEqual(tensor_batch["""labels"""].shape						,(1,)							)







 def _snake_case				(							self							)	->     List[str]:
        _UpperCAmelCase						:		Optional[int]      =				[{"""input_ids""": np.array([0, 1, 2]							)}, {"""input_ids""": np.array([0, 1, 2, 3]							)}]
        _UpperCAmelCase						:		Optional[Any]      =				BertTokenizer.from_pretrained("""bert-base-cased"""							)

        _UpperCAmelCase						:		int      =				tokenizer.pad(_lowerCamelCase						,padding=_lowerCamelCase							)
        self.assertTrue(isinstance(batch["""input_ids"""]						,np.ndarray							)							)
        self.assertEqual(batch["""input_ids"""].tolist()						,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
        _UpperCAmelCase						:		Any      =				tokenizer.pad(_lowerCamelCase						,padding=_lowerCamelCase						,return_tensors="""np"""							)
        self.assertTrue(isinstance(batch["""input_ids"""]						,np.ndarray							)							)
        self.assertEqual(batch["""input_ids"""].tolist()						,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)







 @require_torch
 def _snake_case				(							self							)	->     List[Any]:
        import torch

        _UpperCAmelCase						:		int      =				[{"""input_ids""": torch.tensor([0, 1, 2]							)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]							)}]
        _UpperCAmelCase						:		Any      =				BertTokenizer.from_pretrained("""bert-base-cased"""							)

        _UpperCAmelCase						:		Any      =				tokenizer.pad(_lowerCamelCase						,padding=_lowerCamelCase							)
        self.assertTrue(isinstance(batch["""input_ids"""]						,torch.Tensor							)							)
        self.assertEqual(batch["""input_ids"""].tolist()						,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
        _UpperCAmelCase						:		Any      =				tokenizer.pad(_lowerCamelCase						,padding=_lowerCamelCase						,return_tensors="""pt"""							)
        self.assertTrue(isinstance(batch["""input_ids"""]						,torch.Tensor							)							)
        self.assertEqual(batch["""input_ids"""].tolist()						,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)







 @require_tf
 def _snake_case				(							self							)	->     int:
        import tensorflow as tf

        _UpperCAmelCase						:		Dict      =				[{"""input_ids""": tf.constant([0, 1, 2]							)}, {"""input_ids""": tf.constant([0, 1, 2, 3]							)}]
        _UpperCAmelCase						:		List[Any]      =				BertTokenizer.from_pretrained("""bert-base-cased"""							)

        _UpperCAmelCase						:		Optional[Any]      =				tokenizer.pad(_lowerCamelCase						,padding=_lowerCamelCase							)
        self.assertTrue(isinstance(batch["""input_ids"""]						,tf.Tensor							)							)
        self.assertEqual(batch["""input_ids"""].numpy().tolist()						,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
        _UpperCAmelCase						:		Union[str, Any]      =				tokenizer.pad(_lowerCamelCase						,padding=_lowerCamelCase						,return_tensors="""tf"""							)
        self.assertTrue(isinstance(batch["""input_ids"""]						,tf.Tensor							)							)
        self.assertEqual(batch["""input_ids"""].numpy().tolist()						,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)







 @require_tokenizers
 def _snake_case				(							self							)	->     Tuple:
        _UpperCAmelCase						:		Optional[Any]      =				Tokenizer(WordPiece(unk_token="""[UNK]"""							)							)
        PreTrainedTokenizerFast(tokenizer_object=_lowerCamelCase							)







 @require_tokenizers
 def _snake_case				(							self							)	->     List[Any]:
        _UpperCAmelCase						:		Dict      =				Tokenizer(WordPiece(unk_token="""[UNK]"""							)							)
        with tempfile.TemporaryDirectory() as tmpdirname:
               bert_tokenizer.save(os.path.join(_lowerCamelCase						,"""tokenizer.json"""							)							)
               PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowerCamelCase						,"""tokenizer.json"""							)							)








'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


A_							:				List[Any]						=			{
    """configuration_deta""": ["""DETA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DetaConfig"""],
}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							:				Optional[int]						=			["""DetaImageProcessor"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							:				Optional[Any]						=			[
       """DETA_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """DetaForObjectDetection""",
       """DetaModel""",
       """DetaPreTrainedModel""",
   ]


if TYPE_CHECKING:
   from .configuration_deta import DETA_PRETRAINED_CONFIG_ARCHIVE_MAP, DetaConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .image_processing_deta import DetaImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_deta import (
          DETA_PRETRAINED_MODEL_ARCHIVE_LIST,
          DetaForObjectDetection,
          DetaModel,
          DetaPreTrainedModel,
      )

else:
   import sys

   A_							:				List[Any]						=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



