


'''simple docstring'''



import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


__lowercase			:		Optional[Any] =			logging.getLogger(__name__)






def 						lowerCamelCase							():
					__a    : Any      					=						argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'     )
					parser.add_argument(
					    '--dataset_name'		,   type=_SCREAMING_SNAKE_CASE		,   default='wikitext'		,   help='Name of the training. Explore datasets at: hf.co/datasets.'		,   )
					parser.add_argument(
					    '--dataset_config'		,   type=_SCREAMING_SNAKE_CASE		,   default='wikitext-103-raw-v1'		,   help='Configuration name of the dataset.'     )
					parser.add_argument(
					    '--batch_size'		,   type=_SCREAMING_SNAKE_CASE		,   default=1_000		,   help='Batch size during training.'		,   )
					parser.add_argument(
					    '--vocab_size'		,   type=_SCREAMING_SNAKE_CASE		,   default=10_048		,   help='Size of the desired vocabulary.'		,   )
					parser.add_argument(
					    '--limit'		,   default=_SCREAMING_SNAKE_CASE		,   type=_SCREAMING_SNAKE_CASE		,   help='Limit the number of shards (used for debugging).'		,   )
					parser.add_argument(
					    '--export_to_hub'		,   action='store_true'		,   )

					__a    : Any      					=						parser.parse_args()
					return args






def 						lowerCamelCase							(_SCREAMING_SNAKE_CASE   :     int     ):
					__a    : Any      					=						datasets.load_dataset(args.dataset_name		,   args.dataset_config		,   split='train'     )

					if args.limit is not None:
										__a    : Union[str, Any]      					=						min(len(_SCREAMING_SNAKE_CASE     )		,   args.limit     )
										__a    : int      					=						dataset.select(range(_SCREAMING_SNAKE_CASE     )     )
										logger.info(F"""Limiting the dataset to {args.limit} entries."""     )

					def batch_iterator():
										for i in range(0		,   len(_SCREAMING_SNAKE_CASE     )		,   args.batch_size     ):
															yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
					__a    : List[Any]      					=						Tokenizer(Unigram()     )
					__a    : Union[str, Any]      					=						normalizers.Sequence([normalizers.Replace('``'		,   '"'     ), normalizers.Replace('\'\''		,   '"'     )]     )
					__a    : Optional[int]      					=						pre_tokenizers.Metaspace()

					# Prepare the trainer.
					__a    : Dict      					=						UnigramTrainer(
					    unk_token='<unk>'		,   special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']		,   vocab_size=args.vocab_size		,   )

					logger.info('Training the tokenizer.'     )
					tokenizer.train_from_iterator(batch_iterator()		,   trainer=_SCREAMING_SNAKE_CASE     )
					logger.info('Tokenizer training complete!'     )

					__a    : Optional[int]      					=						tokenizer.token_to_id('[CLS]'     )
					__a    : int      					=						tokenizer.token_to_id('[SEP]'     )
					__a    : List[Any]      					=						processors.TemplateProcessing(
					    single='[CLS]:0 $A:0 [SEP]:0'		,   pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'		,   special_tokens=[
					        ('[CLS]', cls_token_id),
					        ('[SEP]', sep_token_id),
					    ]		,   )
					__a    : int      					=						decoders.Metaspace()

					if args.export_to_hub:
										logger.info('Exporting the trained tokenzier to Hub.'     )
										__a    : Tuple      					=						AlbertTokenizerFast(tokenizer_object=_SCREAMING_SNAKE_CASE     )
										new_tokenizer.push_to_hub('unigram-tokenizer-dataset'     )


if __name__ == "__main__":
	__lowercase			:		Dict =			parse_args()
	main(args)







'''simple docstring'''



def 						lowerCamelCase							(_SCREAMING_SNAKE_CASE   :     set		,   _SCREAMING_SNAKE_CASE   :     set     ):
					assert isinstance(_SCREAMING_SNAKE_CASE		,   _SCREAMING_SNAKE_CASE     ), F"""The input value of [set_a={set_a}] is not a set"""
					assert isinstance(_SCREAMING_SNAKE_CASE		,   _SCREAMING_SNAKE_CASE     ), F"""The input value of [set_b={set_b}] is not a set"""

					return {a + b for a in set_a for b in set_b}


if __name__ == "__main__":
	from doctest import testmod

	testmod()



