code: 
  
  
  import os
  import tempfile
  import unittest
  from typing import List
  
  from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
  from transformers.testing_utils import require_tokenizers
  from transformers.tokenization_utils import PreTrainedTokenizer
  from transformers.tokenization_utils_base import PreTrainedTokenizerBase
  
  
  
  
  
  
  
  @require_tokenizers
  class   lowerCAmelCase__ (       unittest.TestCase):
  
  
  
  
       '''simple docstring'''
       __SCREAMING_SNAKE_CASE						    =				EsmTokenizer
  
  
       def 				_lowerCamelCase				(		self)    ->       Any:
              super().setUp()
              _A   : Tuple       					=				tempfile.mkdtemp()
              # fmt: off
              _A   : List[str]       					=				["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
              # fmt: on
              _A   : Union[str, Any]       					=				os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES["vocab_file"])
              with open(self.vocab_file				,		"w"				,		encoding="utf-8") as vocab_writer:
                     vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
  
  
       def 				_lowerCamelCase				(		self				,		**__lowerCamelCase)    ->       List[PreTrainedTokenizerBase]:
              return [self.get_tokenizer(**__lowerCamelCase)]
  
  
       def 				_lowerCamelCase				(		self				,		**__lowerCamelCase)    ->       PreTrainedTokenizer:
              return self.tokenizer_class.from_pretrained(self.tmpdirname				,		**__lowerCamelCase)
  
  
       def 				_lowerCamelCase				(		self)    ->       Union[str, Any]:
              _A   : Dict       					=				self.tokenizer_class(self.vocab_file)
  
              _A   : Tuple       					=				tokenizer.tokenize("LAGVS")
              self.assertListEqual(__lowerCamelCase				,		["L", "A", "G", "V", "S"])
              self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase)				,		[4, 5, 6, 7, 8])
  
  
       def 				_lowerCamelCase				(		self)    ->       Any:
              _A   : Dict       					=				self.tokenizer_class(self.vocab_file)
  
              _A   : Any       					=				"LAGVS"
              self.assertListEqual(tokenizer.encode(__lowerCamelCase)				,		[0, 4, 5, 6, 7, 8, 2])
  
  
       def 				_lowerCamelCase				(		self)    ->       Tuple:
              _A   : List[str]       					=				self.tokenizer_class(self.vocab_file)
  
              _A   : List[str]       					=				["LAGVS", "WCB"]
              _A   : Optional[Any]       					=				tokenizer(__lowerCamelCase				,		padding=__lowerCamelCase)["input_ids"]
  
              self.assertListEqual(__lowerCamelCase				,		[[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2]])
  
  
       def 				_lowerCamelCase				(		self)    ->       Optional[Any]:
              _A   : Dict       					=				self.tokenizer_class(self.vocab_file)
  
              _A   : int       					=				["LAGVS", "WCB"]
              _A   : List[Any]       					=				tokenizer(__lowerCamelCase				,		padding=__lowerCamelCase)["input_ids"]
  
              self.assertListEqual(__lowerCamelCase				,		[[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2, 1, 1]])
  
  
       def 				_lowerCamelCase				(		self)    ->       List[str]:
              _A   : List[str]       					=				self.get_tokenizers(fast=__lowerCamelCase)
              for tokenizer in tokenizers:
                     with self.subTest(F"{tokenizer.__class__.__name__}"):
                            _A   : Any       					=				"<unk>"
                            _A   : int       					=				"<mask>"
  
                            _A   : int       					=				tokenizer.tokenize(__lowerCamelCase)
                            _A   : Optional[int]       					=				tokenizer.tokenize(__lowerCamelCase)
  
                            self.assertEqual(len(__lowerCamelCase)				,		1)
                            self.assertEqual(len(__lowerCamelCase)				,		1)
                            self.assertEqual(token_a[0]				,		__lowerCamelCase)
                            self.assertEqual(token_a[0]				,		__lowerCamelCase)
  
style_context: 
  
  
  import importlib
  import inspect
  from functools import wraps
  from typing import TYPE_CHECKING, Optional, Union
  
  from .download.streaming_download_manager import (
      xbasename,
      xdirname,
      xet_parse,
      xexists,
      xgetsize,
      xglob,
      xgzip_open,
      xisdir,
      xisfile,
      xjoin,
      xlistdir,
      xnumpy_load,
      xopen,
      xpandas_read_csv,
      xpandas_read_excel,
      xPath,
      xrelpath,
      xsio_loadmat,
      xsplit,
      xsplitext,
      xwalk,
      xxml_dom_minidom_parse,
  )
  from .utils.logging import get_logger
  from .utils.patching import patch_submodule
  from .utils.py_utils import get_imports
  
  
  lowerCAmelCase__   	=				get_logger(__name__)
  
  
  if TYPE_CHECKING:
      from .builder import DatasetBuilder
  def 							_UpperCAmelCase       (UpperCamelCase__  :						Optional[Any]       ,							UpperCamelCase__  :						Optional[Union[str, bool]] = None		):
         _A   : Dict       					=				importlib.import_module(UpperCamelCase__		)
  
         # TODO(QL): always update the module to add subsequent new authentication
         if hasattr(UpperCamelCase__       ,							"_patched_for_streaming"		) and module._patched_for_streaming:
                return
  
         def wrap_auth(UpperCamelCase__  :						Optional[int]		):
                @wraps(UpperCamelCase__		)
                def wrapper(*UpperCamelCase__  :						str       ,							**UpperCamelCase__  :						Optional[int]		):
                       return function(*UpperCamelCase__       ,							use_auth_token=UpperCamelCase__       ,							**UpperCamelCase__		)
  
                _A   : List[Any]       					=				"wrap_auth"
                return wrapper
  
         # open files in a streaming fashion
         patch_submodule(UpperCamelCase__       ,							"open"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"os.listdir"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"os.walk"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"glob.glob"       ,							wrap_auth(UpperCamelCase__		)		).start()
         # allow to navigate in remote zip files
         patch_submodule(UpperCamelCase__       ,							"os.path.join"       ,							UpperCamelCase__		).start()
         patch_submodule(UpperCamelCase__       ,							"os.path.dirname"       ,							UpperCamelCase__		).start()
         patch_submodule(UpperCamelCase__       ,							"os.path.basename"       ,							UpperCamelCase__		).start()
         patch_submodule(UpperCamelCase__       ,							"os.path.relpath"       ,							UpperCamelCase__		).start()
         patch_submodule(UpperCamelCase__       ,							"os.path.split"       ,							UpperCamelCase__		).start()
         patch_submodule(UpperCamelCase__       ,							"os.path.splitext"       ,							UpperCamelCase__		).start()
         # allow checks on paths
         patch_submodule(UpperCamelCase__       ,							"os.path.exists"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"os.path.isdir"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"os.path.isfile"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"os.path.getsize"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"pathlib.Path"       ,							UpperCamelCase__		).start()
         # file readers
         patch_submodule(UpperCamelCase__       ,							"gzip.open"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"numpy.load"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"pandas.read_csv"       ,							wrap_auth(UpperCamelCase__		)       ,							attrs=["__version__"]		).start()
         patch_submodule(UpperCamelCase__       ,							"pandas.read_excel"       ,							wrap_auth(UpperCamelCase__		)       ,							attrs=["__version__"]		).start()
         patch_submodule(UpperCamelCase__       ,							"scipy.io.loadmat"       ,							wrap_auth(UpperCamelCase__		)       ,							attrs=["__version__"]		).start()
         patch_submodule(UpperCamelCase__       ,							"xml.etree.ElementTree.parse"       ,							wrap_auth(UpperCamelCase__		)		).start()
         patch_submodule(UpperCamelCase__       ,							"xml.dom.minidom.parse"       ,							wrap_auth(UpperCamelCase__		)		).start()
         _A   : Any       					=				True
  
  
  
  
  
  
  def 							_UpperCAmelCase       (UpperCamelCase__  :						"DatasetBuilder"		):
         extend_module_for_streaming(builder.__module__       ,							use_auth_token=builder.use_auth_token		)
         # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
         if not builder.__module__.startswith("datasets."		):  # check that it's not a packaged builder like csv
                for imports in get_imports(inspect.getfile(builder.__class__		)		):
                       if imports[0] == "internal":
                              _A   : Optional[Any]       					=				imports[1]
                              _A   : int       					=				".".join(builder.__module__.split("."		)[:-1] + [internal_import_name]		)
                              extend_module_for_streaming(UpperCamelCase__       ,							use_auth_token=builder.use_auth_token		)
  
      # builders can inherit from other builders that might use streaming functionality
      # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
      # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
         from .builder import DatasetBuilder
  
         _A   : Tuple       					=				[
             cls.__module__
             for cls in type(UpperCamelCase__		).__mro__[1:]  # make sure it's not the same module we've already patched
             if issubclass(cls       ,							UpperCamelCase__		) and cls.__module__ != DatasetBuilder.__module__
         ]  # check it's not a standard builder from datasets.builder
         for module in parent_builder_modules:
                extend_module_for_streaming(UpperCamelCase__       ,							use_auth_token=builder.use_auth_token		)
  