



'''simple docstring'''


import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


lowercase											=      logging.getLogger(__name__)






def 			UpperCAmelCase_							(		):


     '''simple docstring'''







     a_									=argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."							)
     parser.add_argument(
         "--dataset_name"							,						type=__A							,						default="wikitext"							,						help="Name of the training. Explore datasets at: hf.co/datasets."							,						)
     parser.add_argument(
         "--dataset_config"							,						type=__A							,						default="wikitext-103-raw-v1"							,						help="Configuration name of the dataset."							)
     parser.add_argument(
         "--batch_size"							,						type=__A							,						default=1_0_0_0							,						help="Batch size during training."							,						)
     parser.add_argument(
         "--vocab_size"							,						type=__A							,						default=1_0_0_4_8							,						help="Size of the desired vocabulary."							,						)
     parser.add_argument(
         "--limit"							,						default=__A							,						type=__A							,						help="Limit the number of shards (used for debugging)."							,						)
     parser.add_argument(
         "--export_to_hub"							,						action="store_true"							,						)

     a_									=parser.parse_args()
     return args






def 			UpperCAmelCase_							(		lowercase__							):


     '''simple docstring'''







     a_									=datasets.load_dataset(args.dataset_name							,						args.dataset_config							,						split="train"							)

     if args.limit is not None:
          a_									=min(len(__A							)							,						args.limit							)
          a_									=dataset.select(range(__A							)							)
          logger.info(F"""Limiting the dataset to {args.limit} entries."""							)

     def batch_iterator():
          for i in range(0							,						len(__A							)							,						args.batch_size							):
               yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
     a_									=Tokenizer(Unigram()							)
     a_									=normalizers.Sequence([normalizers.Replace("``"							,						"\""							), normalizers.Replace("''"							,						"\""							)]							)
     a_									=pre_tokenizers.Metaspace()

     # Prepare the trainer.
     a_									=UnigramTrainer(
         unk_token="<unk>"							,						special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]							,						vocab_size=args.vocab_size							,						)

     logger.info("Training the tokenizer."							)
     tokenizer.train_from_iterator(batch_iterator()							,						trainer=__A							)
     logger.info("Tokenizer training complete!"							)

     a_									=tokenizer.token_to_id("[CLS]"							)
     a_									=tokenizer.token_to_id("[SEP]"							)
     a_									=processors.TemplateProcessing(
         single="[CLS]:0 $A:0 [SEP]:0"							,						pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"							,						special_tokens=[
             ("[CLS]", cls_token_id),
             ("[SEP]", sep_token_id),
         ]							,						)
     a_									=decoders.Metaspace()

     if args.export_to_hub:
          logger.info("Exporting the trained tokenzier to Hub."							)
          a_									=AlbertTokenizerFast(tokenizer_object=__A							)
          new_tokenizer.push_to_hub("unigram-tokenizer-dataset"							)


if __name__ == "__main__":
  lowercase											=      parse_args()
  main(args)






'''simple docstring'''







import math
from timeit import timeit







def _UpperCamelCase			(	__A       )				->						int:


      '''simple docstring'''



      UpperCamelCase__      =							0
      UpperCamelCase__      =							abs(__A       )
      while True:
            UpperCamelCase__      =							n // 10
            digits += 1
            if n == 0:
                  break
      return digits







def _UpperCamelCase			(	__A       )				->						int:


      '''simple docstring'''



      return 1 if n == 0 else math.floor(math.log(abs(__A       ) ,     10       ) + 1       )







def _UpperCamelCase			(	__A       )				->						int:


      '''simple docstring'''



      return len(str(abs(__A       )       )       )







def _UpperCamelCase			(	)				->						None:


      '''simple docstring'''



      from collections.abc import Callable

      def benchmark_a_function(__A ,     __A       ) -> None:
            UpperCamelCase__      =							F'''{func.__name__}({value})'''
            UpperCamelCase__      =							timeit(F'''__main__.{call}''' ,     setup="import __main__"       )
            print(F'''{call}: {func(__A       )} -- {timing} seconds'''       )

      for value in (262144, 1125899906842624, 1267650600228229401496703205376):
            for func in (num_digits, num_digits_fast, num_digits_faster):
                  benchmark_a_function(__A ,     __A       )
            print()


if __name__ == "__main__":
  import doctest

  doctest.testmod()
  benchmark()





