code: 
  
  
  
  import json
  import os
  import tempfile
  
  import datasets
  from datasets.arrow_writer import ArrowWriter
  from datasets.features import ArrayaD
  from utils import generate_examples, get_duration
  
  
  _lowerCAmelCase       :      int							=							(30, 4_87)
  _lowerCAmelCase       :      Tuple							=							(36, 10_24)
  _lowerCAmelCase       :      Dict							=							(1_00, 1_00)
  _lowerCAmelCase       :      int							=							1_00
  
  _lowerCAmelCase       :      Optional[int]							=							datasets.Features(
      {"text": ArrayaD(SHAPE_TEST_1, dtype="float32"), "image": ArrayaD(SHAPE_TEST_2, dtype="float32")}
  )
  
  _lowerCAmelCase,		_lowerCAmelCase       :      int							=							os.path.split(__file__)
  _lowerCAmelCase       :      Optional[Any]							=							os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))
  
  @get_duration
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		int		,						SCREAMING_SNAKE_CASE__			:		int		,						SCREAMING_SNAKE_CASE__			:		Optional[Any]       )						->     List[str]:
  
  
  
  
  
       '''simple docstring'''
  
       with ArrowWriter(features=SCREAMING_SNAKE_CASE__		,						path=os.path.join(SCREAMING_SNAKE_CASE__		,						"beta.arrow"       )       ) as writer:
            for key, record in dummy_data:
                 _UpperCAmelCase    :			Optional[Any]              =					my_features.encode_example(SCREAMING_SNAKE_CASE__       )
                 writer.write(SCREAMING_SNAKE_CASE__       )
            _UpperCAmelCase    :			List[str]              =					writer.finalize()
  
  @get_duration
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		Any		,						SCREAMING_SNAKE_CASE__			:		int       )						->     int:
  
  
  
  
  
       '''simple docstring'''
  
       _UpperCAmelCase    :			List[Any]              =					datasets.Dataset.from_file(
           filename=os.path.join(SCREAMING_SNAKE_CASE__		,						"beta.arrow"       )		,						info=datasets.DatasetInfo(features=SCREAMING_SNAKE_CASE__       )       )
       for _ in dataset:
            pass
  
  @get_duration
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		Tuple		,						SCREAMING_SNAKE_CASE__			:		Union[str, Any]       )						->     Dict:
  
  
  
  
  
       '''simple docstring'''
  
       _UpperCAmelCase    :			Any              =					datasets.Dataset.from_file(
           filename=os.path.join(SCREAMING_SNAKE_CASE__		,						"beta.arrow"       )		,						info=datasets.DatasetInfo(features=SCREAMING_SNAKE_CASE__       )       )
       dataset.set_format("numpy"       )
       for _ in dataset:
            pass
  
  @get_duration
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		Optional[int]		,						SCREAMING_SNAKE_CASE__			:		Union[str, Any]       )						->     Optional[Any]:
  
  
  
  
  
       '''simple docstring'''
  
       _UpperCAmelCase    :			List[str]              =					10
       _UpperCAmelCase    :			int              =					datasets.Dataset.from_file(
           filename=os.path.join(SCREAMING_SNAKE_CASE__		,						"beta.arrow"       )		,						info=datasets.DatasetInfo(features=SCREAMING_SNAKE_CASE__       )       )
       for i in range(0		,						len(SCREAMING_SNAKE_CASE__       )		,						SCREAMING_SNAKE_CASE__       ):
            _UpperCAmelCase    :			int              =					dataset[i : i + batch_size]
  
  @get_duration
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		Union[str, Any]		,						SCREAMING_SNAKE_CASE__			:		Any       )						->     Tuple:
  
  
  
  
  
       '''simple docstring'''
  
       _UpperCAmelCase    :			str              =					10
       _UpperCAmelCase    :			int              =					datasets.Dataset.from_file(
           filename=os.path.join(SCREAMING_SNAKE_CASE__		,						"beta.arrow"       )		,						info=datasets.DatasetInfo(features=SCREAMING_SNAKE_CASE__       )       )
       dataset.set_format("numpy"       )
       for i in range(0		,						len(SCREAMING_SNAKE_CASE__       )		,						SCREAMING_SNAKE_CASE__       ):
            _UpperCAmelCase    :			Optional[Any]              =					dataset[i : i + batch_size]
  
  @get_duration
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		Any		,						SCREAMING_SNAKE_CASE__			:		Tuple       )						->     Dict:
  
  
  
  
  
       '''simple docstring'''
  
       _UpperCAmelCase    :			List[str]              =					datasets.Dataset.from_file(
           filename=os.path.join(SCREAMING_SNAKE_CASE__		,						"beta.arrow"       )		,						info=datasets.DatasetInfo(features=SCREAMING_SNAKE_CASE__       )       )
       for col in feats:
            _UpperCAmelCase    :			str              =					dataset[col]
  
  @get_duration
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		Any		,						SCREAMING_SNAKE_CASE__			:		List[str]       )						->     Tuple:
  
  
  
  
  
       '''simple docstring'''
  
       _UpperCAmelCase    :			str              =					datasets.Dataset.from_file(
           filename=os.path.join(SCREAMING_SNAKE_CASE__		,						"beta.arrow"       )		,						info=datasets.DatasetInfo(features=SCREAMING_SNAKE_CASE__       )       )
       dataset.set_format("numpy"       )
       for col in feats:
            _UpperCAmelCase    :			Tuple              =					dataset[col]
  
  def 		__snake_case		(   )						->     Optional[Any]:
  
  
  
  
  
       '''simple docstring'''
  
       _UpperCAmelCase    :			List[str]              =					{}
       _UpperCAmelCase    :			Dict              =					(
           read_unformated,
           read_formatted_as_numpy,
           read_batch_unformated,
           read_batch_formatted_as_numpy,
           read_col_unformated,
           read_col_formatted_as_numpy,
       )
       with tempfile.TemporaryDirectory() as tmp_dir:
            _UpperCAmelCase    :			Any              =					datasets.Features({"image": ArrayaD(SCREAMING_SNAKE_CASE__		,						dtype="float32"       )}       )
            _UpperCAmelCase    :			List[Any]              =					generate_examples(features=SCREAMING_SNAKE_CASE__		,						num_examples=SCREAMING_SNAKE_CASE__       )
            _UpperCAmelCase    :			Any              =					write(SCREAMING_SNAKE_CASE__		,						SCREAMING_SNAKE_CASE__		,						SCREAMING_SNAKE_CASE__       )
            for read_func in read_functions:
                 _UpperCAmelCase    :			Any              =					read_func(SCREAMING_SNAKE_CASE__		,						SCREAMING_SNAKE_CASE__       )
  
       with tempfile.TemporaryDirectory() as tmp_dir:
            # don't use fixed length for fair comparison
            # feats = datasets.Features(
            #     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
            # )
            _UpperCAmelCase    :			str              =					datasets.Features({"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"       )       )       )}       )
            _UpperCAmelCase    :			Dict              =					generate_examples(
                features=SCREAMING_SNAKE_CASE__		,						num_examples=SCREAMING_SNAKE_CASE__		,						seq_shapes={"image": SPEED_TEST_SHAPE}       )
            _UpperCAmelCase    :			Union[str, Any]              =					write(SCREAMING_SNAKE_CASE__		,						SCREAMING_SNAKE_CASE__		,						SCREAMING_SNAKE_CASE__       )
            for read_func in read_functions:
                 _UpperCAmelCase    :			Dict              =					read_func(SCREAMING_SNAKE_CASE__		,						SCREAMING_SNAKE_CASE__       )
  
       with tempfile.TemporaryDirectory() as tmp_dir:
            # don't use fixed length for fair comparison
            # feats = datasets.Features(
            #     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
            # )
            _UpperCAmelCase    :			Optional[int]              =					datasets.Features({"image": datasets.Sequence(datasets.Value("float32"       )       )}       )
            _UpperCAmelCase    :			Optional[int]              =					generate_examples(
                features=SCREAMING_SNAKE_CASE__		,						num_examples=SCREAMING_SNAKE_CASE__		,						seq_shapes={"image": [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}		,						)
            _UpperCAmelCase    :			Any              =					write(SCREAMING_SNAKE_CASE__		,						SCREAMING_SNAKE_CASE__		,						SCREAMING_SNAKE_CASE__       )
            for read_func in read_functions:
                 _UpperCAmelCase    :			Any              =					read_func(SCREAMING_SNAKE_CASE__		,						SCREAMING_SNAKE_CASE__       )
  
       with open(SCREAMING_SNAKE_CASE__		,						"wb"       ) as f:
            f.write(json.dumps(SCREAMING_SNAKE_CASE__       ).encode("utf-8"       )       )
  
  
  if __name__ == "__main__":  # useful to run the profiler
   benchmark_array_xd()
  
style_context: 
  
  
  
  """simple docstring"""
  
  
  
  
  from arguments import TokenizerTrainingArguments
  from datasets import load_dataset
  from tqdm import tqdm
  
  from transformers import AutoTokenizer, HfArgumentParser
  from transformers.models.gpta.tokenization_gpta import bytes_to_unicode
  
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		str=10       )						->     List[Any]:
  
  
  
  
  
       '''simple docstring'''
  
       for _ in tqdm(range(0		,						args.n_examples		,						SCREAMING_SNAKE_CASE__       )       ):
            yield [next(SCREAMING_SNAKE_CASE__       )[args.text_column] for _ in range(SCREAMING_SNAKE_CASE__       )]
  
  
  # Configuration
  _lowerCAmelCase       :      Optional[Any]							=							HfArgumentParser(TokenizerTrainingArguments)
  _lowerCAmelCase       :      Dict							=							parser.parse_args()
  
  # Base tokenizer
  _lowerCAmelCase       :      int							=							AutoTokenizer.from_pretrained(args.base_tokenizer)
  _lowerCAmelCase       :      List[str]							=							list(bytes_to_unicode().values())
  
  # Load dataset
  _lowerCAmelCase       :      Dict							=							load_dataset(args.dataset_name, split="train", streaming=True)
  _lowerCAmelCase       :      List[Any]							=							iter(dataset)
  
  
  # Training and saving
  _lowerCAmelCase       :      List[str]							=							tokenizer.train_new_from_iterator(
      batch_iterator(), vocab_size=args.vocab_size, initial_alphabet=base_vocab
  )
  new_tokenizer.save_pretrained(args.tokenizer_name, push_to_hub=args.push_to_hub)
  