code: 
  
  import multiprocessing
  import os
  from typing import BinaryIO, Optional, Union
  
  from .. import Dataset, Features, NamedSplit, config
  from ..formatting import query_table
  from ..packaged_modules.csv.csv import Csv
  from ..utils import logging
  from ..utils.typing import NestedDataStructureLike, PathLike
  from .abc import AbstractDatasetReader
  
  
  
  
  
  
  class     lowerCamelCase_			(  _UpperCAmelCase  ):
  
  
  
  
  	def __init__(    self	:    List[str]				,			__A	:    str				,			__A	:    Any = None				,			__A	:    List[Any] = None				,			__A	:    Dict = None				,			__A	:    Dict = False				,			__A	:    Optional[Any] = False				,			__A	:    str = None				,			**__A	:    str				,			):
  		super().__init__(
  		    __A				,			split=__A				,			features=__A				,			cache_dir=__A				,			keep_in_memory=__A				,			streaming=__A				,			num_proc=__A				,			**__A				,			)
  		__A   : Optional[int]								= path_or_paths if isinstance(__A				,			__A       ) else {self.split: path_or_paths}
  		__A   : List[Any]								= Csv(
  		    cache_dir=__A				,			data_files=__A				,			features=__A				,			**__A				,			)
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    str       ):
  
  
  
  
  
  
  		if self.streaming:
  			__A   : str								= self.builder.as_streaming_dataset(split=self.split       )
  		# Build regular (map-style) dataset
  		else:
  			__A   : Optional[Any]								= None
  			__A   : int								= None
  			__A   : Any								= None
  			__A   : Dict								= None
  
  			self.builder.download_and_prepare(
  			    download_config=__A				,			download_mode=__A				,			verification_mode=__A				,			base_path=__A				,			num_proc=self.num_proc				,			)
  			__A   : Tuple								= self.builder.as_dataset(
  			    split=self.split				,			verification_mode=__A				,			in_memory=self.keep_in_memory       )
  		return dataset
  
  
  
  
  
  
  class     lowerCamelCase_			:
  
  
  
  
  	def __init__(    self	:    List[str]				,			__A	:    Any				,			__A	:    str				,			__A	:    List[str] = None				,			__A	:    Tuple = None				,			**__A	:    Union[str, Any]				,			):
  
  
  
  
  
  
  		if num_proc is not None and num_proc <= 0:
  			raise ValueError(F"""num_proc {num_proc} must be an integer > 0."""       )
  
  		__A   : List[Any]								= dataset
  		__A   : Optional[int]								= path_or_buf
  		__A   : int								= batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
  		__A   : Tuple								= num_proc
  		__A   : int								= """utf-8"""
  		__A   : Optional[int]								= to_csv_kwargs
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int       ):
  		__A   : Dict								= self.to_csv_kwargs.pop("""path_or_buf"""				,			__A       )
  		__A   : Dict								= self.to_csv_kwargs.pop("""header"""				,			__A       )
  		__A   : Union[str, Any]								= self.to_csv_kwargs.pop("""index"""				,			__A       )
  
  		if isinstance(self.path_or_buf				,			(str, bytes, os.PathLike)       ):
  			with open(self.path_or_buf				,			"""wb+"""       ) as buffer:
  				__A   : Dict								= self._write(file_obj=__A				,			header=__A				,			index=__A				,			**self.to_csv_kwargs       )
  		else:
  			__A   : Tuple								= self._write(file_obj=self.path_or_buf				,			header=__A				,			index=__A				,			**self.to_csv_kwargs       )
  		return written
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[Any]				,			__A	:    List[Any]       ):
  		__A						,							__A						,							__A						,							__A   : Any								= args
  
  		__A   : List[str]								= query_table(
  		    table=self.dataset.data				,			key=slice(__A				,			offset + self.batch_size       )				,			indices=self.dataset._indices				,			)
  		__A   : List[Any]								= batch.to_pandas().to_csv(
  		    path_or_buf=__A				,			header=header if (offset == 0) else False				,			index=__A				,			**__A       )
  		return csv_str.encode(self.encoding       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Any				,			__A	:    Optional[int]				,			__A	:    Any				,			__A	:    List[Any]				,			**__A	:    str       ):
  		__A   : Union[str, Any]								= 0
  
  		if self.num_proc is None or self.num_proc == 1:
  			for offset in logging.tqdm(
  			    range(0				,			len(self.dataset       )				,			self.batch_size       )				,			unit="""ba"""				,			disable=not logging.is_progress_bar_enabled()				,			desc="""Creating CSV from Arrow format"""				,			):
  				__A   : Any								= self._batch_csv((offset, header, index, to_csv_kwargs)       )
  				written += file_obj.write(__A       )
  
  		else:
  			__A						,							__A   : str								= len(self.dataset       ), self.batch_size
  			with multiprocessing.Pool(self.num_proc       ) as pool:
  				for csv_str in logging.tqdm(
  				    pool.imap(
  				        self._batch_csv				,			[(offset, header, index, to_csv_kwargs) for offset in range(0				,			__A				,			__A       )]				,			)				,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size				,			unit="""ba"""				,			disable=not logging.is_progress_bar_enabled()				,			desc="""Creating CSV from Arrow format"""				,			):
  					written += file_obj.write(__A       )
  
  		return written
  
  
  
  
  
  
style_context: 
  
  import os
  import tempfile
  import unittest
  from typing import List
  
  from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
  from transformers.testing_utils import require_tokenizers
  from transformers.tokenization_utils import PreTrainedTokenizer
  from transformers.tokenization_utils_base import PreTrainedTokenizerBase
  
  
  
  
  
  
  @require_tokenizers
  class     lowerCamelCase_			(  unittest.TestCase  ):
  	_lowercase		:      List[str]    				=							EsmTokenizer
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		super().setUp()
  		__A   : Dict								= tempfile.mkdtemp()
  		# fmt: off
  		__A   : List[str]								= ["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
  		# fmt: on
  		__A   : int								= os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES["""vocab_file"""]       )
  		with open(self.vocab_file				,			"""w"""				,			encoding="""utf-8"""       ) as vocab_writer:
  			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict				,			**__A	:    int       ):
  		return [self.get_tokenizer(**__A       )]
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			**__A	:    List[Any]       ):
  		return self.tokenizer_class.from_pretrained(self.tmpdirname				,			**__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]       ):
  		__A   : Union[str, Any]								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : int								= tokenizer.tokenize("""LAGVS"""       )
  		self.assertListEqual(__A				,			["""L""", """A""", """G""", """V""", """S"""]       )
  		self.assertListEqual(tokenizer.convert_tokens_to_ids(__A       )				,			[4, 5, 6, 7, 8]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    str       ):
  		__A   : Union[str, Any]								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : int								= """LAGVS"""
  		self.assertListEqual(tokenizer.encode(__A       )				,			[0, 4, 5, 6, 7, 8, 2]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict       ):
  		__A   : str								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : Tuple								= ["""LAGVS""", """WCB"""]
  		__A   : str								= tokenizer(__A				,			padding=__A       )["""input_ids"""]
  
  		self.assertListEqual(__A				,			[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		__A   : Dict								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : Any								= ["""LAGVS""", """WCB"""]
  		__A   : str								= tokenizer(__A				,			padding=__A       )["""input_ids"""]
  
  		self.assertListEqual(__A				,			[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]       ):
  		__A   : Optional[int]								= self.get_tokenizers(fast=__A       )
  		for tokenizer in tokenizers:
  			with self.subTest(F"""{tokenizer.__class__.__name__}"""       ):
  				__A   : Any								= """<unk>"""
  				__A   : str								= """<mask>"""
  
  				__A   : Optional[Any]								= tokenizer.tokenize(__A       )
  				__A   : str								= tokenizer.tokenize(__A       )
  
  				self.assertEqual(len(__A       )				,			1       )
  				self.assertEqual(len(__A       )				,			1       )
  				self.assertEqual(token_a[0]				,			__A       )
  				self.assertEqual(token_a[0]				,			__A       )
  
  
  
  
  
  