code: 
  import os
  import tempfile
  from pathlib import Path
  from unittest import TestCase
  
  import pyarrow as pa
  import pytest
  
  from datasets.arrow_dataset import Dataset
  from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
  from datasets.info import DatasetInfo
  from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo
  
  from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
  
  
  
  
  
  class    _snake_case					(  snake_case       ):
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a=False	):
  				__magic_name__	:	List[Any]			     =  (
  				    filename_skip_take["filename"],
  				    filename_skip_take["skip"] if "skip" in filename_skip_take else None,
  				    filename_skip_take["take"] if "take" in filename_skip_take else None,
  				)
  				open(os.path.join(_a	)		,     "wb"	).close()
  				__magic_name__	:	str			     =  pa.Table.from_pydict({"filename": [Path(_a	).name] * 100}	)
  				if take == -1:
  					__magic_name__	:	Dict			     =  len(_a	) - skip
  				if skip is not None and take is not None:
  					__magic_name__	:	List[Any]			     =  pa_table.slice(_a		,     _a	)
  				return pa_table
  
  
  
  
  
  
  
  class    _snake_case					(  snake_case       ):
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	str			     =  "my_name"
  				__magic_name__	:	List[str]			     =  SplitInfo(name="train"		,     num_examples=100	)
  				__magic_name__	:	Dict			     =  SplitInfo(name="test"		,     num_examples=100	)
  				__magic_name__	:	str			     =  [train_info, test_info]
  				__magic_name__	:	int			     =  SplitDict()
  				split_dict.add(_a	)
  				split_dict.add(_a	)
  				__magic_name__	:	Any			     =  DatasetInfo(splits=_a	)
  
  				with tempfile.TemporaryDirectory() as tmp_dir:
  					__magic_name__	:	str			     =  ReaderTest(_a		,     _a	)
  
  					__magic_name__	:	Dict			     =  "test[:33%]"
  					__magic_name__	:	int			     =  Dataset(**reader.read(_a		,     _a		,     _a	)	)
  					self.assertEqual(dset["filename"][0]		,     f'''{name}-test'''	)
  					self.assertEqual(dset.num_rows		,     33	)
  					self.assertEqual(dset.num_columns		,     1	)
  
  					__magic_name__	:	Any			     =  ["train", "test[:33%]"]
  					__magic_name__	:	Dict			     =  [Split.TRAIN, ReadInstruction.from_spec("test[:33%]"	)]
  					for instructions in [instructionsa, instructionsa]:
  						__magic_name__	:	Tuple			     =  [reader.read(_a		,     _a		,     _a	) for instr in instructions]
  						__magic_name__	:	Optional[int]			     =  (Dataset(**_a	) for dataset_kwargs in datasets_kwargs)
  						self.assertEqual(train_dset["filename"][0]		,     f'''{name}-train'''	)
  						self.assertEqual(train_dset.num_rows		,     100	)
  						self.assertEqual(train_dset.num_columns		,     1	)
  						self.assertIsInstance(train_dset.split		,     _a	)
  						self.assertEqual(str(train_dset.split	)		,     "train"	)
  						self.assertEqual(test_dset["filename"][0]		,     f'''{name}-test'''	)
  						self.assertEqual(test_dset.num_rows		,     33	)
  						self.assertEqual(test_dset.num_columns		,     1	)
  						self.assertIsInstance(test_dset.split		,     _a	)
  						self.assertEqual(str(test_dset.split	)		,     "test[:33%]"	)
  						del train_dset, test_dset
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	List[Any]			     =  "my_name"
  				__magic_name__	:	Dict			     =  SplitInfo(name="train"		,     num_examples=1_000		,     shard_lengths=[100] * 10	)
  				__magic_name__	:	str			     =  [train_info]
  				__magic_name__	:	List[str]			     =  SplitDict()
  				split_dict.add(_a	)
  				__magic_name__	:	str			     =  DatasetInfo(splits=_a	)
  
  				with tempfile.TemporaryDirectory() as tmp_dir:
  					__magic_name__	:	Union[str, Any]			     =  ReaderTest(_a		,     _a	)
  
  					__magic_name__	:	Any			     =  "train[:33%]"
  					__magic_name__	:	str			     =  Dataset(**reader.read(_a		,     _a		,     _a	)	)
  					self.assertEqual(dset["filename"][0]		,     f'''{name}-train-00000-of-00010'''	)
  					self.assertEqual(dset["filename"][-1]		,     f'''{name}-train-00003-of-00010'''	)
  					self.assertEqual(dset.num_rows		,     330	)
  					self.assertEqual(dset.num_columns		,     1	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Optional[Any]			     =  SplitInfo(name="train"		,     num_examples=100	)
  				__magic_name__	:	Union[str, Any]			     =  SplitInfo(name="test"		,     num_examples=100	)
  				__magic_name__	:	List[str]			     =  SplitDict()
  				split_dict.add(_a	)
  				split_dict.add(_a	)
  				__magic_name__	:	Optional[Any]			     =  DatasetInfo(splits=_a	)
  
  				with tempfile.TemporaryDirectory() as tmp_dir:
  					__magic_name__	:	Any			     =  ReaderTest(_a		,     _a	)
  
  					__magic_name__	:	Optional[int]			     =  [
  					    {"filename": os.path.join(_a		,     "train"	)},
  					    {"filename": os.path.join(_a		,     "test"	), "skip": 10, "take": 10},
  					]
  					__magic_name__	:	Optional[int]			     =  Dataset(**reader.read_files(_a		,     original_instructions="train+test[10:20]"	)	)
  					self.assertEqual(dset.num_rows		,     110	)
  					self.assertEqual(dset.num_columns		,     1	)
  					del dset
  
  
  
  
  
  
  
  @pytest.mark.parametrize("in_memory"   ,     [False, True]     )
  def 						lowerCAmelCase_		(      _snake_case   :						Tuple   ,     _snake_case   :						Tuple   ,     _snake_case   :						Tuple     )       ->	List[str]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	Dict			     =  arrow_file
  	with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
  		__magic_name__	:	int			     =  ArrowReader.read_table(_snake_case   ,     in_memory=_snake_case     )
  	assert table.shape == dataset.data.shape
  	assert set(table.column_names     ) == set(dataset.data.column_names     )
  	assert dict(table.to_pydict()     ) == dict(dataset.data.to_pydict()     )  # to_pydict returns OrderedDict
  
  
  
  
  
  
  @pytest.mark.parametrize("in_memory"   ,     [False, True]     )
  def 						lowerCAmelCase_		(      _snake_case   :						Tuple   ,     _snake_case   :						List[str]   ,     _snake_case   :						Dict     )       ->	Union[str, Any]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	Dict			     =  arrow_file
  	__magic_name__	:	Tuple			     =  ArrowReader(""   ,     _snake_case     )
  	with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
  		__magic_name__	:	Optional[int]			     =  reader.read_files([{"filename": filename}]   ,     in_memory=_snake_case     )
  	assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
  	__magic_name__	:	List[str]			     =  dataset_kwargs["arrow_table"]
  	assert table.shape == dataset.data.shape
  	assert set(table.column_names     ) == set(dataset.data.column_names     )
  	assert dict(table.to_pydict()     ) == dict(dataset.data.to_pydict()     )  # to_pydict returns OrderedDict
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      )       ->	Optional[Any]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	assert ReadInstruction("train"   ,     to=10   ,     unit="abs"     ).to_spec() == "train[:10]"
  	assert ReadInstruction("train"   ,     from_=-80   ,     to=10   ,     unit="%"     ).to_spec() == "train[-80%:10%]"
  
  	__magic_name__	:	Optional[Any]			     =  "train+test"
  	assert ReadInstruction.from_spec(_snake_case     ).to_spec() == spec_train_test
  
  	__magic_name__	:	Tuple			     =  "train[2:10]"
  	assert ReadInstruction.from_spec(_snake_case     ).to_spec() == spec_train_abs
  
  	__magic_name__	:	Optional[int]			     =  "train[15%:-20%]"
  	assert ReadInstruction.from_spec(_snake_case     ).to_spec() == spec_train_pct
  
  	__magic_name__	:	List[Any]			     =  "train[:10%](closest)"
  	assert ReadInstruction.from_spec(_snake_case     ).to_spec() == "train[:10%]"
  
  	__magic_name__	:	Dict			     =  "train[:10%](pct1_dropremainder)"
  	assert ReadInstruction.from_spec(_snake_case     ).to_spec() == spec_train_pct_rounding
  
  	__magic_name__	:	int			     =  "train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)"
  	assert ReadInstruction.from_spec(_snake_case     ).to_spec() == spec_train_test_pct_rounding
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      )       ->	int:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	List[Any]			     =  "dummy"
  	__magic_name__	:	List[Any]			     =  [SplitInfo(name="train"   ,     num_examples=100     )]
  	__magic_name__	:	int			     =  "train[:33%]"
  	__magic_name__	:	int			     =  "arrow"
  	__magic_name__	:	Optional[int]			     =  "prefix"
  
  	__magic_name__	:	List[str]			     =  make_file_instructions(_snake_case   ,     _snake_case   ,     _snake_case   ,     _snake_case   ,     _snake_case     )
  	assert isinstance(_snake_case   ,     _snake_case     )
  	assert file_instructions.num_examples == 33
  	assert file_instructions.file_instructions == [
  	    {"filename": os.path.join(_snake_case   ,     F'''{name}-train.arrow'''     ), "skip": 0, "take": 33}
  	]
  
  	__magic_name__	:	List[Any]			     =  [SplitInfo(name="train"   ,     num_examples=100   ,     shard_lengths=[10] * 10     )]
  	__magic_name__	:	Optional[int]			     =  make_file_instructions(_snake_case   ,     _snake_case   ,     _snake_case   ,     _snake_case   ,     _snake_case     )
  	assert isinstance(_snake_case   ,     _snake_case     )
  	assert file_instructions.num_examples == 33
  	assert file_instructions.file_instructions == [
  	    {"filename": os.path.join(_snake_case   ,     F'''{name}-train-00000-of-00010.arrow'''     ), "skip": 0, "take": -1},
  	    {"filename": os.path.join(_snake_case   ,     F'''{name}-train-00001-of-00010.arrow'''     ), "skip": 0, "take": -1},
  	    {"filename": os.path.join(_snake_case   ,     F'''{name}-train-00002-of-00010.arrow'''     ), "skip": 0, "take": -1},
  	    {"filename": os.path.join(_snake_case   ,     F'''{name}-train-00003-of-00010.arrow'''     ), "skip": 0, "take": 3},
  	]
  
  
  
  
  
  
  @pytest.mark.parametrize("name, expected_exception"   ,     [(None, TypeError), ("", ValueError)]     )
  def 						lowerCAmelCase_		(      _snake_case   :						Union[str, Any]   ,     _snake_case   :						Tuple     )       ->	int:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	Optional[Any]			     =  [SplitInfo(name="train"   ,     num_examples=100     )]
  	__magic_name__	:	int			     =  "train"
  	__magic_name__	:	str			     =  "arrow"
  	__magic_name__	:	Any			     =  "prefix_path"
  	with pytest.raises(_snake_case     ):
  		__magic_name__	:	List[Any]			     =  make_file_instructions(_snake_case   ,     _snake_case   ,     _snake_case   ,     _snake_case   ,     _snake_case     )
  
style_context: 
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils import AddedToken
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  
  
  if is_sentencepiece_available():
  				from .tokenization_xlm_roberta import XLMRobertaTokenizer
  else:
  				snake_case				:	Any							   =						None
  
  
  snake_case				:	Union[str, Any]							   =						logging.get_logger(__name__)
  
  snake_case				:	Any							   =						{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}
  
  snake_case				:	Optional[Any]							   =						{
      "vocab_file": {
          "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
          "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
          "xlm-roberta-large-finetuned-conll02-dutch": (
              "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
          ),
          "xlm-roberta-large-finetuned-conll02-spanish": (
              "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
          ),
          "xlm-roberta-large-finetuned-conll03-english": (
              "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
          ),
          "xlm-roberta-large-finetuned-conll03-german": (
              "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
          ),
      },
      "tokenizer_file": {
          "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json",
          "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json",
          "xlm-roberta-large-finetuned-conll02-dutch": (
              "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json"
          ),
          "xlm-roberta-large-finetuned-conll02-spanish": (
              "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json"
          ),
          "xlm-roberta-large-finetuned-conll03-english": (
              "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json"
          ),
          "xlm-roberta-large-finetuned-conll03-german": (
              "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json"
          ),
      },
  }
  
  snake_case				:	Optional[int]							   =						{
      "xlm-roberta-base": 512,
      "xlm-roberta-large": 512,
      "xlm-roberta-large-finetuned-conll02-dutch": 512,
      "xlm-roberta-large-finetuned-conll02-spanish": 512,
      "xlm-roberta-large-finetuned-conll03-english": 512,
      "xlm-roberta-large-finetuned-conll03-german": 512,
  }
  
  
  
  
  
  class    _snake_case      (  snake_case       ):
  			UpperCamelCase__					   =					VOCAB_FILES_NAMES
  			UpperCamelCase__					   =					PRETRAINED_VOCAB_FILES_MAP
  			UpperCamelCase__					   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  			UpperCamelCase__					   =					['input_ids', 'attention_mask']
  			UpperCamelCase__					   =					XLMRobertaTokenizer
  
  
  
  
  
  
  
  			def __init__(       self		,     _a=None		,     _a=None		,     _a="<s>"		,     _a="</s>"		,     _a="</s>"		,     _a="<s>"		,     _a="<unk>"		,     _a="<pad>"		,     _a="<mask>"		,     **_a		,     ):
  				# Mask token behave like a normal word, i.e. include the space before it
  				__magic_name__	:	List[str]			     =  AddedToken(_a		,     lstrip=_a		,     rstrip=_a	) if isinstance(_a		,     _a	) else mask_token
  
  				super().__init__(
  				    _a		,     tokenizer_file=_a		,     bos_token=_a		,     eos_token=_a		,     sep_token=_a		,     cls_token=_a		,     unk_token=_a		,     pad_token=_a		,     mask_token=_a		,     **_a		,     )
  
  				__magic_name__	:	Optional[int]			     =  vocab_file
  				__magic_name__	:	List[str]			     =  False if not self.vocab_file else True
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a = None	):
  
  				if token_ids_a is None:
  					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  				__magic_name__	:	Union[str, Any]			     =  [self.cls_token_id]
  				__magic_name__	:	str			     =  [self.sep_token_id]
  				return cls + token_ids_a + sep + sep + token_ids_a + sep
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a = None	):
  				__magic_name__	:	List[str]			     =  [self.sep_token_id]
  				__magic_name__	:	List[str]			     =  [self.cls_token_id]
  
  				if token_ids_a is None:
  					return len(cls + token_ids_a + sep	) * [0]
  				return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a = None	):
  				if not self.can_save_slow_tokenizer:
  					raise ValueError(
  					    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
  					    "tokenizer."	)
  
  				if not os.path.isdir(_a	):
  					logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''	)
  					return
  				__magic_name__	:	Any			     =  os.path.join(
  				    _a		,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)
  
  				if os.path.abspath(self.vocab_file	) != os.path.abspath(_a	):
  					copyfile(self.vocab_file		,     _a	)
  
  				return (out_vocab_file,)
  