code: 
  
  
  import multiprocessing
  import os
  from typing import BinaryIO, Optional, Union
  
  from .. import Dataset, Features, NamedSplit, config
  from ..formatting import query_table
  from ..packaged_modules.csv.csv import Csv
  from ..utils import logging
  from ..utils.typing import NestedDataStructureLike, PathLike
  from .abc import AbstractDatasetReader
  
  
  class 		_SCREAMING_SNAKE_CASE    (	A    ):
  
  
       def __init__(     self       ,				A_       ,				A_ = None       ,				A_ = None       ,				A_ = None       ,				A_ = False       ,				A_ = False       ,				A_ = None       ,				**A_       ,				):
         super().__init__(
             A_       ,				split=A_       ,				features=A_       ,				cache_dir=A_       ,				keep_in_memory=A_       ,				streaming=A_       ,				num_proc=A_       ,				**A_       ,				)
         _UpperCAmelCase		:       Optional[Any]       =    path_or_paths if isinstance(A_       ,				A_						) else {self.split: path_or_paths}
         _UpperCAmelCase		:       List[str]       =    Csv(
             cache_dir=A_       ,				data_files=A_       ,				features=A_       ,				**A_       ,				)
  
  
  
  
       def 					__snake_case(     self						):
         # Build iterable dataset
         if self.streaming:
           _UpperCAmelCase		:       Union[str, Any]       =    self.builder.as_streaming_dataset(split=self.split						)
         # Build regular (map-style) dataset
         else:
           _UpperCAmelCase		:       Optional[int]       =    None
           _UpperCAmelCase		:       List[Any]       =    None
           _UpperCAmelCase		:       Dict       =    None
           _UpperCAmelCase		:       Dict       =    None
  
           self.builder.download_and_prepare(
               download_config=A_       ,				download_mode=A_       ,				verification_mode=A_       ,				base_path=A_       ,				num_proc=self.num_proc       ,				)
           _UpperCAmelCase		:       Union[str, Any]       =    self.builder.as_dataset(
               split=self.split       ,				verification_mode=A_       ,				in_memory=self.keep_in_memory						)
         return dataset
  
  
  class 		_SCREAMING_SNAKE_CASE    :
  
  
       def __init__(     self       ,				A_       ,				A_       ,				A_ = None       ,				A_ = None       ,				**A_       ,				):
         if num_proc is not None and num_proc <= 0:
           raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''						)
  
         _UpperCAmelCase		:       Union[str, Any]       =    dataset
         _UpperCAmelCase		:       Tuple       =    path_or_buf
         _UpperCAmelCase		:       Optional[int]       =    batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         _UpperCAmelCase		:       str       =    num_proc
         _UpperCAmelCase		:       Any       =    """utf-8"""
         _UpperCAmelCase		:       Optional[int]       =    to_csv_kwargs
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Tuple       =    self.to_csv_kwargs.pop("""path_or_buf"""       ,				A_						)
         _UpperCAmelCase		:       List[str]       =    self.to_csv_kwargs.pop("""header"""       ,				A_						)
         _UpperCAmelCase		:       Dict       =    self.to_csv_kwargs.pop("""index"""       ,				A_						)
  
         if isinstance(self.path_or_buf       ,				(str, bytes, os.PathLike)						):
           with open(self.path_or_buf       ,				"""wb+"""						) as buffer:
             _UpperCAmelCase		:       str       =    self._write(file_obj=A_       ,				header=A_       ,				index=A_       ,				**self.to_csv_kwargs						)
         else:
           _UpperCAmelCase		:       List[Any]       =    self._write(file_obj=self.path_or_buf       ,				header=A_       ,				index=A_       ,				**self.to_csv_kwargs						)
         return written
  
  
       def 					__snake_case(     self       ,				A_						):
         _UpperCAmelCase,_UpperCAmelCase,_UpperCAmelCase,_UpperCAmelCase		:       Optional[int]       =    args
  
         _UpperCAmelCase		:       Optional[int]       =    query_table(
             table=self.dataset.data       ,				key=slice(A_       ,				offset + self.batch_size						)       ,				indices=self.dataset._indices       ,				)
         _UpperCAmelCase		:       Optional[Any]       =    batch.to_pandas().to_csv(
             path_or_buf=A_       ,				header=header if (offset == 0) else False       ,				index=A_       ,				**A_						)
         return csv_str.encode(self.encoding						)
  
  
  
  
       def 					__snake_case(     self       ,				A_       ,				A_       ,				A_       ,				**A_						):
         _UpperCAmelCase		:       Tuple       =    0
  
         if self.num_proc is None or self.num_proc == 1:
           for offset in logging.tqdm(
               range(0       ,				len(self.dataset						)       ,				self.batch_size						)       ,				unit="""ba"""       ,				disable=not logging.is_progress_bar_enabled()       ,				desc="""Creating CSV from Arrow format"""       ,				):
             _UpperCAmelCase		:       int       =    self._batch_csv((offset, header, index, to_csv_kwargs)						)
             written += file_obj.write(A_						)
  
         else:
           _UpperCAmelCase,_UpperCAmelCase		:       str       =    len(self.dataset						), self.batch_size
           with multiprocessing.Pool(self.num_proc						) as pool:
             for csv_str in logging.tqdm(
                 pool.imap(
                     self._batch_csv       ,				[(offset, header, index, to_csv_kwargs) for offset in range(0       ,				A_       ,				A_						)]       ,				)       ,				total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,				unit="""ba"""       ,				disable=not logging.is_progress_bar_enabled()       ,				desc="""Creating CSV from Arrow format"""       ,				):
               written += file_obj.write(A_						)
  
         return written
  
  
  
  
  
  
style_context: 
  
  
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils import AddedToken
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  
  
  if is_sentencepiece_available():
   from .tokenization_xlm_roberta import XLMRobertaTokenizer
  else:
   SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    None
  
  
  SCREAMING_SNAKE_CASE__				:    Dict         =    logging.get_logger(__name__)
  
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}
  
  SCREAMING_SNAKE_CASE__				:    Any         =    {
      'vocab_file': {
          'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
          'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
          'xlm-roberta-large-finetuned-conll02-dutch': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
          ),
          'xlm-roberta-large-finetuned-conll02-spanish': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
          ),
          'xlm-roberta-large-finetuned-conll03-english': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
          ),
          'xlm-roberta-large-finetuned-conll03-german': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
          ),
      },
      'tokenizer_file': {
          'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
          'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
          'xlm-roberta-large-finetuned-conll02-dutch': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
          ),
          'xlm-roberta-large-finetuned-conll02-spanish': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
          ),
          'xlm-roberta-large-finetuned-conll03-english': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
          ),
          'xlm-roberta-large-finetuned-conll03-german': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
          ),
      },
  }
  
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    {
      'xlm-roberta-base': 512,
      'xlm-roberta-large': 512,
      'xlm-roberta-large-finetuned-conll02-dutch': 512,
      'xlm-roberta-large-finetuned-conll02-spanish': 512,
      'xlm-roberta-large-finetuned-conll03-english': 512,
      'xlm-roberta-large-finetuned-conll03-german': 512,
  }
  
  
  class 		_SCREAMING_SNAKE_CASE    (	A    ):
       __SCREAMING_SNAKE_CASE    				=						VOCAB_FILES_NAMES
       __SCREAMING_SNAKE_CASE    				=						PRETRAINED_VOCAB_FILES_MAP
       __SCREAMING_SNAKE_CASE    				=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __SCREAMING_SNAKE_CASE    				=						['''input_ids''', '''attention_mask''']
       __SCREAMING_SNAKE_CASE    				=						XLMRobertaTokenizer
  
  
       def __init__(     self       ,				A_=None       ,				A_=None       ,				A_="<s>"       ,				A_="</s>"       ,				A_="</s>"       ,				A_="<s>"       ,				A_="<unk>"       ,				A_="<pad>"       ,				A_="<mask>"       ,				**A_       ,				):
         # Mask token behave like a normal word, i.e. include the space before it
         _UpperCAmelCase		:       int       =    AddedToken(A_       ,				lstrip=A_       ,				rstrip=A_						) if isinstance(A_       ,				A_						) else mask_token
  
         super().__init__(
             A_       ,				tokenizer_file=A_       ,				bos_token=A_       ,				eos_token=A_       ,				sep_token=A_       ,				cls_token=A_       ,				unk_token=A_       ,				pad_token=A_       ,				mask_token=A_       ,				**A_       ,				)
  
         _UpperCAmelCase		:       Union[str, Any]       =    vocab_file
         _UpperCAmelCase		:       List[str]       =    False if not self.vocab_file else True
  
  
       def 					__snake_case(     self       ,				A_       ,				A_ = None						):
  
         if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         _UpperCAmelCase		:       Optional[Any]       =    [self.cls_token_id]
         _UpperCAmelCase		:       Union[str, Any]       =    [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep
  
  
       def 					__snake_case(     self       ,				A_       ,				A_ = None						):
         _UpperCAmelCase		:       Tuple       =    [self.sep_token_id]
         _UpperCAmelCase		:       Optional[int]       =    [self.cls_token_id]
  
         if token_ids_a is None:
           return len(cls + token_ids_a + sep						) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]
  
  
  
  
       def 					__snake_case(     self       ,				A_       ,				A_ = None						):
         if not self.can_save_slow_tokenizer:
           raise ValueError(
               """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
               """tokenizer."""						)
  
         if not os.path.isdir(A_						):
           logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''						)
           return
         _UpperCAmelCase		:       List[str]       =    os.path.join(
             A_       ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
  
         if os.path.abspath(self.vocab_file						) != os.path.abspath(A_						):
           copyfile(self.vocab_file       ,				A_						)
  
         return (out_vocab_file,)
  
  
  
  
  
  