


import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase







@require_tokenizers
class   lowerCAmelCase__ (       unittest.TestCase):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE						    =				EsmTokenizer


     def 				_lowerCamelCase				(		self)    ->       Any:
            super().setUp()
            _A   : Tuple       					=				tempfile.mkdtemp()
            # fmt: off
            _A   : List[str]       					=				["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
            # fmt: on
            _A   : Union[str, Any]       					=				os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES["vocab_file"])
            with open(self.vocab_file				,		"w"				,		encoding="utf-8") as vocab_writer:
                   vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))


     def 				_lowerCamelCase				(		self				,		**__lowerCamelCase)    ->       List[PreTrainedTokenizerBase]:
            return [self.get_tokenizer(**__lowerCamelCase)]


     def 				_lowerCamelCase				(		self				,		**__lowerCamelCase)    ->       PreTrainedTokenizer:
            return self.tokenizer_class.from_pretrained(self.tmpdirname				,		**__lowerCamelCase)


     def 				_lowerCamelCase				(		self)    ->       Union[str, Any]:
            _A   : Dict       					=				self.tokenizer_class(self.vocab_file)

            _A   : Tuple       					=				tokenizer.tokenize("LAGVS")
            self.assertListEqual(__lowerCamelCase				,		["L", "A", "G", "V", "S"])
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase)				,		[4, 5, 6, 7, 8])


     def 				_lowerCamelCase				(		self)    ->       Any:
            _A   : Dict       					=				self.tokenizer_class(self.vocab_file)

            _A   : Any       					=				"LAGVS"
            self.assertListEqual(tokenizer.encode(__lowerCamelCase)				,		[0, 4, 5, 6, 7, 8, 2])


     def 				_lowerCamelCase				(		self)    ->       Tuple:
            _A   : List[str]       					=				self.tokenizer_class(self.vocab_file)

            _A   : List[str]       					=				["LAGVS", "WCB"]
            _A   : Optional[Any]       					=				tokenizer(__lowerCamelCase				,		padding=__lowerCamelCase)["input_ids"]

            self.assertListEqual(__lowerCamelCase				,		[[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2]])


     def 				_lowerCamelCase				(		self)    ->       Optional[Any]:
            _A   : Dict       					=				self.tokenizer_class(self.vocab_file)

            _A   : int       					=				["LAGVS", "WCB"]
            _A   : List[Any]       					=				tokenizer(__lowerCamelCase				,		padding=__lowerCamelCase)["input_ids"]

            self.assertListEqual(__lowerCamelCase				,		[[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2, 1, 1]])


     def 				_lowerCamelCase				(		self)    ->       List[str]:
            _A   : List[str]       					=				self.get_tokenizers(fast=__lowerCamelCase)
            for tokenizer in tokenizers:
                   with self.subTest(F"{tokenizer.__class__.__name__}"):
                          _A   : Any       					=				"<unk>"
                          _A   : int       					=				"<mask>"

                          _A   : int       					=				tokenizer.tokenize(__lowerCamelCase)
                          _A   : Optional[int]       					=				tokenizer.tokenize(__lowerCamelCase)

                          self.assertEqual(len(__lowerCamelCase)				,		1)
                          self.assertEqual(len(__lowerCamelCase)				,		1)
                          self.assertEqual(token_a[0]				,		__lowerCamelCase)
                          self.assertEqual(token_a[0]				,		__lowerCamelCase)




import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


lowerCAmelCase__   	=				get_logger(__name__)


if TYPE_CHECKING:
    from .builder import DatasetBuilder
def 							_UpperCAmelCase       (UpperCamelCase__  :						Optional[Any]       ,							UpperCamelCase__  :						Optional[Union[str, bool]] = None		):
       _A   : Dict       					=				importlib.import_module(UpperCamelCase__		)

       # TODO(QL): always update the module to add subsequent new authentication
       if hasattr(UpperCamelCase__       ,							"_patched_for_streaming"		) and module._patched_for_streaming:
              return

       def wrap_auth(UpperCamelCase__  :						Optional[int]		):
              @wraps(UpperCamelCase__		)
              def wrapper(*UpperCamelCase__  :						str       ,							**UpperCamelCase__  :						Optional[int]		):
                     return function(*UpperCamelCase__       ,							use_auth_token=UpperCamelCase__       ,							**UpperCamelCase__		)

              _A   : List[Any]       					=				"wrap_auth"
              return wrapper

       # open files in a streaming fashion
       patch_submodule(UpperCamelCase__       ,							"open"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.listdir"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.walk"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"glob.glob"       ,							wrap_auth(UpperCamelCase__		)		).start()
       # allow to navigate in remote zip files
       patch_submodule(UpperCamelCase__       ,							"os.path.join"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.dirname"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.basename"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.relpath"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.split"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.splitext"       ,							UpperCamelCase__		).start()
       # allow checks on paths
       patch_submodule(UpperCamelCase__       ,							"os.path.exists"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.isdir"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.isfile"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.getsize"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"pathlib.Path"       ,							UpperCamelCase__		).start()
       # file readers
       patch_submodule(UpperCamelCase__       ,							"gzip.open"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"numpy.load"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"pandas.read_csv"       ,							wrap_auth(UpperCamelCase__		)       ,							attrs=["__version__"]		).start()
       patch_submodule(UpperCamelCase__       ,							"pandas.read_excel"       ,							wrap_auth(UpperCamelCase__		)       ,							attrs=["__version__"]		).start()
       patch_submodule(UpperCamelCase__       ,							"scipy.io.loadmat"       ,							wrap_auth(UpperCamelCase__		)       ,							attrs=["__version__"]		).start()
       patch_submodule(UpperCamelCase__       ,							"xml.etree.ElementTree.parse"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"xml.dom.minidom.parse"       ,							wrap_auth(UpperCamelCase__		)		).start()
       _A   : Any       					=				True






def 							_UpperCAmelCase       (UpperCamelCase__  :						"DatasetBuilder"		):
       extend_module_for_streaming(builder.__module__       ,							use_auth_token=builder.use_auth_token		)
       # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
       if not builder.__module__.startswith("datasets."		):  # check that it's not a packaged builder like csv
              for imports in get_imports(inspect.getfile(builder.__class__		)		):
                     if imports[0] == "internal":
                            _A   : Optional[Any]       					=				imports[1]
                            _A   : int       					=				".".join(builder.__module__.split("."		)[:-1] + [internal_import_name]		)
                            extend_module_for_streaming(UpperCamelCase__       ,							use_auth_token=builder.use_auth_token		)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
       from .builder import DatasetBuilder

       _A   : Tuple       					=				[
           cls.__module__
           for cls in type(UpperCamelCase__		).__mro__[1:]  # make sure it's not the same module we've already patched
           if issubclass(cls       ,							UpperCamelCase__		) and cls.__module__ != DatasetBuilder.__module__
       ]  # check it's not a standard builder from datasets.builder
       for module in parent_builder_modules:
              extend_module_for_streaming(UpperCamelCase__       ,							use_auth_token=builder.use_auth_token		)
