
import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece

class 		lowercase_       (       unittest.TestCase				):

 def lowerCamelCase_				(      self     ,	__UpperCamelCase     ):




        """simple docstring"""


        UpperCamelCase_							       =       list(tokenizer_class.max_model_input_sizes.keys()     )
        for model_name in sa_models[:1]:
               UpperCamelCase_							       =       tokenizer_class.from_pretrained(__UpperCamelCase     )
               self.assertIsNotNone(__UpperCamelCase     )
               self.assertIsInstance(__UpperCamelCase     ,	__UpperCamelCase     )
               self.assertIsInstance(__UpperCamelCase     ,	__UpperCamelCase     )

               for special_tok in tokenizer.all_special_tokens:
                      self.assertIsInstance(__UpperCamelCase     ,	__UpperCamelCase     )
                      UpperCamelCase_							       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
                      self.assertIsInstance(__UpperCamelCase     ,	__UpperCamelCase     )

 def lowerCamelCase_				(      self     ,	__UpperCamelCase     ,	__UpperCamelCase = None     ):




        """simple docstring"""


        UpperCamelCase_							       =       pickle.dumps(__UpperCamelCase     )
        self.assertIsNotNone(__UpperCamelCase     )

        UpperCamelCase_							       =       pickle.loads(__UpperCamelCase     )

        # Ensure is_fast is correctly restored
        self.assertEqual(be_restored.is_fast     ,	be_original.is_fast     )

        # Ensure encodings are potentially correctly restored
        if be_original.is_fast:
               self.assertIsNotNone(be_restored.encodings     )
        else:
               self.assertIsNone(be_restored.encodings     )

        # Ensure the keys are the same
        for original_v, restored_v in zip(be_original.values()     ,	be_restored.values()     ):
               if equal_op:
                      self.assertTrue(equal_op(__UpperCamelCase     ,	__UpperCamelCase     )     )
               else:
                      self.assertEqual(__UpperCamelCase     ,	__UpperCamelCase     )

 @slow
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        self.check_tokenizer_from_pretrained(__UpperCamelCase     )

 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        self.assertEqual(TensorType("""tf"""     )     ,	TensorType.TENSORFLOW     )
        self.assertEqual(TensorType("""pt"""     )     ,	TensorType.PYTORCH     )
        self.assertEqual(TensorType("""np"""     )     ,	TensorType.NUMPY     )

 @require_tokenizers
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        import numpy as np

        UpperCamelCase_							       =       BertTokenizer.from_pretrained("""bert-base-cased"""     )
        UpperCamelCase_							       =       BertTokenizerFast.from_pretrained("""bert-base-cased"""     )

        # Python no tensor
        with self.subTest("""BatchEncoding (Python, return_tensors=None)"""     ):
               self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""     )     )

        with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""     ,	return_tensors=TensorType.NUMPY     )     ,	np.array_equal     )

        with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""     ):
               self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""     )     )

        with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""     ,	return_tensors=TensorType.NUMPY     )     ,	np.array_equal     )

 @require_tf
 @require_tokenizers
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        import tensorflow as tf

        def tf_array_equals(__UpperCamelCase     ,	__UpperCamelCase     ):
               return tf.reduce_all(tf.equal(__UpperCamelCase     ,	__UpperCamelCase     )     )

        UpperCamelCase_							       =       BertTokenizer.from_pretrained("""bert-base-cased"""     )
        UpperCamelCase_							       =       BertTokenizerFast.from_pretrained("""bert-base-cased"""     )

        with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""     ,	return_tensors=TensorType.TENSORFLOW     )     ,	__UpperCamelCase     )

        with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""     ,	return_tensors=TensorType.TENSORFLOW     )     ,	__UpperCamelCase     )

 @require_torch
 @require_tokenizers
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        import torch

        UpperCamelCase_							       =       BertTokenizer.from_pretrained("""bert-base-cased"""     )
        UpperCamelCase_							       =       BertTokenizerFast.from_pretrained("""bert-base-cased"""     )

        with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""     ,	return_tensors=TensorType.PYTORCH     )     ,	torch.equal     )

        with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""     ,	return_tensors=TensorType.PYTORCH     )     ,	torch.equal     )

 @require_tokenizers
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        UpperCamelCase_							       =       BertTokenizer.from_pretrained("""bert-base-cased"""     )
        UpperCamelCase_							       =       BertTokenizerFast.from_pretrained("""bert-base-cased"""     )

        with self.subTest("""Python Tokenizer"""     ):
               self.assertFalse(tokenizer_p("""Small example to_encode"""     ).is_fast     )

        with self.subTest("""Rust Tokenizer"""     ):
               self.assertTrue(tokenizer_r("""Small example to_encode"""     ).is_fast     )

 @require_tokenizers
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        UpperCamelCase_							       =       BertTokenizerFast.from_pretrained("""bert-base-cased"""     )
        UpperCamelCase_							       =       tokenizer_r(["""Test""", """\xad""", """test"""]     ,	is_split_into_words=__UpperCamelCase     )

        self.assertEqual(encoded.word_to_tokens(0     )     ,	TokenSpan(start=1     ,	end=2     )     )
        self.assertEqual(encoded.word_to_tokens(1     )     ,	__UpperCamelCase     )
        self.assertEqual(encoded.word_to_tokens(2     )     ,	TokenSpan(start=2     ,	end=3     )     )

 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        UpperCamelCase_							       =       BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}     )
        UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""np"""     )
        self.assertEqual(tensor_batch["""inputs"""].shape     ,	(2, 3)     )
        self.assertEqual(tensor_batch["""labels"""].shape     ,	(2,)     )
        # test converting the converted
        with CaptureStderr() as cs:
               UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""np"""     )
        self.assertFalse(len(cs.err     )     ,	msg=f'''should have no warning, but got {cs.err}'''     )

        UpperCamelCase_							       =       BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}     )
        UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""np"""     ,	prepend_batch_axis=__UpperCamelCase     )
        self.assertEqual(tensor_batch["""inputs"""].shape     ,	(1, 3)     )
        self.assertEqual(tensor_batch["""labels"""].shape     ,	(1,)     )

 @require_torch
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        UpperCamelCase_							       =       BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}     )
        UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""pt"""     )
        self.assertEqual(tensor_batch["""inputs"""].shape     ,	(2, 3)     )
        self.assertEqual(tensor_batch["""labels"""].shape     ,	(2,)     )
        # test converting the converted
        with CaptureStderr() as cs:
               UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""pt"""     )
        self.assertFalse(len(cs.err     )     ,	msg=f'''should have no warning, but got {cs.err}'''     )

        UpperCamelCase_							       =       BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}     )
        UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""pt"""     ,	prepend_batch_axis=__UpperCamelCase     )
        self.assertEqual(tensor_batch["""inputs"""].shape     ,	(1, 3)     )
        self.assertEqual(tensor_batch["""labels"""].shape     ,	(1,)     )

 @require_tf
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        UpperCamelCase_							       =       BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}     )
        UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""tf"""     )
        self.assertEqual(tensor_batch["""inputs"""].shape     ,	(2, 3)     )
        self.assertEqual(tensor_batch["""labels"""].shape     ,	(2,)     )
        # test converting the converted
        with CaptureStderr() as cs:
               UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""tf"""     )
        self.assertFalse(len(cs.err     )     ,	msg=f'''should have no warning, but got {cs.err}'''     )

        UpperCamelCase_							       =       BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}     )
        UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""tf"""     ,	prepend_batch_axis=__UpperCamelCase     )
        self.assertEqual(tensor_batch["""inputs"""].shape     ,	(1, 3)     )
        self.assertEqual(tensor_batch["""labels"""].shape     ,	(1,)     )

 @require_flax
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        UpperCamelCase_							       =       BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}     )
        UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""jax"""     )
        self.assertEqual(tensor_batch["""inputs"""].shape     ,	(2, 3)     )
        self.assertEqual(tensor_batch["""labels"""].shape     ,	(2,)     )
        # test converting the converted
        with CaptureStderr() as cs:
               UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""jax"""     )
        self.assertFalse(len(cs.err     )     ,	msg=f'''should have no warning, but got {cs.err}'''     )

        UpperCamelCase_							       =       BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}     )
        UpperCamelCase_							       =       batch.convert_to_tensors(tensor_type="""jax"""     ,	prepend_batch_axis=__UpperCamelCase     )
        self.assertEqual(tensor_batch["""inputs"""].shape     ,	(1, 3)     )
        self.assertEqual(tensor_batch["""labels"""].shape     ,	(1,)     )

 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        UpperCamelCase_							       =       [{"""input_ids""": np.array([0, 1, 2]     )}, {"""input_ids""": np.array([0, 1, 2, 3]     )}]
        UpperCamelCase_							       =       BertTokenizer.from_pretrained("""bert-base-cased"""     )

        UpperCamelCase_							       =       tokenizer.pad(__UpperCamelCase     ,	padding=__UpperCamelCase     )
        self.assertTrue(isinstance(batch["""input_ids"""]     ,	np.ndarray     )     )
        self.assertEqual(batch["""input_ids"""].tolist()     ,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
        UpperCamelCase_							       =       tokenizer.pad(__UpperCamelCase     ,	padding=__UpperCamelCase     ,	return_tensors="""np"""     )
        self.assertTrue(isinstance(batch["""input_ids"""]     ,	np.ndarray     )     )
        self.assertEqual(batch["""input_ids"""].tolist()     ,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )

 @require_torch
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        import torch

        UpperCamelCase_							       =       [{"""input_ids""": torch.tensor([0, 1, 2]     )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]     )}]
        UpperCamelCase_							       =       BertTokenizer.from_pretrained("""bert-base-cased"""     )

        UpperCamelCase_							       =       tokenizer.pad(__UpperCamelCase     ,	padding=__UpperCamelCase     )
        self.assertTrue(isinstance(batch["""input_ids"""]     ,	torch.Tensor     )     )
        self.assertEqual(batch["""input_ids"""].tolist()     ,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
        UpperCamelCase_							       =       tokenizer.pad(__UpperCamelCase     ,	padding=__UpperCamelCase     ,	return_tensors="""pt"""     )
        self.assertTrue(isinstance(batch["""input_ids"""]     ,	torch.Tensor     )     )
        self.assertEqual(batch["""input_ids"""].tolist()     ,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )

 @require_tf
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        import tensorflow as tf

        UpperCamelCase_							       =       [{"""input_ids""": tf.constant([0, 1, 2]     )}, {"""input_ids""": tf.constant([0, 1, 2, 3]     )}]
        UpperCamelCase_							       =       BertTokenizer.from_pretrained("""bert-base-cased"""     )

        UpperCamelCase_							       =       tokenizer.pad(__UpperCamelCase     ,	padding=__UpperCamelCase     )
        self.assertTrue(isinstance(batch["""input_ids"""]     ,	tf.Tensor     )     )
        self.assertEqual(batch["""input_ids"""].numpy().tolist()     ,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
        UpperCamelCase_							       =       tokenizer.pad(__UpperCamelCase     ,	padding=__UpperCamelCase     ,	return_tensors="""tf"""     )
        self.assertTrue(isinstance(batch["""input_ids"""]     ,	tf.Tensor     )     )
        self.assertEqual(batch["""input_ids"""].numpy().tolist()     ,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )

 @require_tokenizers
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        UpperCamelCase_							       =       Tokenizer(WordPiece(unk_token="""[UNK]"""     )     )
        PreTrainedTokenizerFast(tokenizer_object=__UpperCamelCase     )

 @require_tokenizers
 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        UpperCamelCase_							       =       Tokenizer(WordPiece(unk_token="""[UNK]"""     )     )
        with tempfile.TemporaryDirectory() as tmpdirname:
               bert_tokenizer.save(os.path.join(__UpperCamelCase     ,	"""tokenizer.json"""     )     )
               PreTrainedTokenizerFast(tokenizer_file=os.path.join(__UpperCamelCase     ,	"""tokenizer.json"""     )     )





import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


_A	  =					get_logger(__name__)


if TYPE_CHECKING:
      from .builder import DatasetBuilder




def 				lowerCamelCase__				(						a__  :  int	,      a__  :  Optional[Union[str, bool]] = None     )		->	Dict:
       UpperCamelCase_							       =       importlib.import_module(a__     )

       # TODO(QL): always update the module to add subsequent new authentication
       if hasattr(a__	,      """_patched_for_streaming"""     ) and module._patched_for_streaming:
              return

       def wrap_auth(a__  :  int     ):
              @wraps(a__     )
              def wrapper(*a__  :  int	,      **a__  :  List[str]     ):
                     return function(*a__	,      use_auth_token=a__	,      **a__     )

              UpperCamelCase_							       =       """wrap_auth"""
              return wrapper

       # open files in a streaming fashion
       patch_submodule(a__	,      """open"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """os.listdir"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """os.walk"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """glob.glob"""	,      wrap_auth(a__     )     ).start()
       # allow to navigate in remote zip files
       patch_submodule(a__	,      """os.path.join"""	,      a__     ).start()
       patch_submodule(a__	,      """os.path.dirname"""	,      a__     ).start()
       patch_submodule(a__	,      """os.path.basename"""	,      a__     ).start()
       patch_submodule(a__	,      """os.path.relpath"""	,      a__     ).start()
       patch_submodule(a__	,      """os.path.split"""	,      a__     ).start()
       patch_submodule(a__	,      """os.path.splitext"""	,      a__     ).start()
       # allow checks on paths
       patch_submodule(a__	,      """os.path.exists"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """os.path.isdir"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """os.path.isfile"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """os.path.getsize"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """pathlib.Path"""	,      a__     ).start()
       # file readers
       patch_submodule(a__	,      """gzip.open"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """numpy.load"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """pandas.read_csv"""	,      wrap_auth(a__     )	,      attrs=["""__version__"""]     ).start()
       patch_submodule(a__	,      """pandas.read_excel"""	,      wrap_auth(a__     )	,      attrs=["""__version__"""]     ).start()
       patch_submodule(a__	,      """scipy.io.loadmat"""	,      wrap_auth(a__     )	,      attrs=["""__version__"""]     ).start()
       patch_submodule(a__	,      """xml.etree.ElementTree.parse"""	,      wrap_auth(a__     )     ).start()
       patch_submodule(a__	,      """xml.dom.minidom.parse"""	,      wrap_auth(a__     )     ).start()
       UpperCamelCase_							       =       True







def 				lowerCamelCase__				(						a__  :  "DatasetBuilder"     )		->	Tuple:
       extend_module_for_streaming(builder.__module__	,      use_auth_token=builder.use_auth_token     )
       # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
       if not builder.__module__.startswith("""datasets."""     ):  # check that it's not a packaged builder like csv
              for imports in get_imports(inspect.getfile(builder.__class__     )     ):
                     if imports[0] == "internal":
                            UpperCamelCase_							       =       imports[1]
                            UpperCamelCase_							       =       """.""".join(builder.__module__.split("""."""     )[:-1] + [internal_import_name]     )
                            extend_module_for_streaming(a__	,      use_auth_token=builder.use_auth_token     )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
       from .builder import DatasetBuilder

       UpperCamelCase_							       =       [
           cls.__module__
           for cls in type(a__     ).__mro__[1:]  # make sure it's not the same module we've already patched
           if issubclass(cls	,      a__     ) and cls.__module__ != DatasetBuilder.__module__
       ]  # check it's not a standard builder from datasets.builder
       for module in parent_builder_modules:
              extend_module_for_streaming(a__	,      use_auth_token=builder.use_auth_token     )



