code: 
  
  
  '''simple docstring'''
  
  
  
  import argparse
  import logging
  
  import datasets
  from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
  from tokenizers.models import Unigram
  from tokenizers.trainers import UnigramTrainer
  
  from transformers import AlbertTokenizerFast
  
  
  __lowercase			:		Optional[Any] =			logging.getLogger(__name__)
  
  
  
  
  
  
  def 						lowerCamelCase							():
  					__a    : Any      					=						argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'     )
  					parser.add_argument(
  					    '--dataset_name'		,   type=_SCREAMING_SNAKE_CASE		,   default='wikitext'		,   help='Name of the training. Explore datasets at: hf.co/datasets.'		,   )
  					parser.add_argument(
  					    '--dataset_config'		,   type=_SCREAMING_SNAKE_CASE		,   default='wikitext-103-raw-v1'		,   help='Configuration name of the dataset.'     )
  					parser.add_argument(
  					    '--batch_size'		,   type=_SCREAMING_SNAKE_CASE		,   default=1_000		,   help='Batch size during training.'		,   )
  					parser.add_argument(
  					    '--vocab_size'		,   type=_SCREAMING_SNAKE_CASE		,   default=10_048		,   help='Size of the desired vocabulary.'		,   )
  					parser.add_argument(
  					    '--limit'		,   default=_SCREAMING_SNAKE_CASE		,   type=_SCREAMING_SNAKE_CASE		,   help='Limit the number of shards (used for debugging).'		,   )
  					parser.add_argument(
  					    '--export_to_hub'		,   action='store_true'		,   )
  
  					__a    : Any      					=						parser.parse_args()
  					return args
  
  
  
  
  
  
  def 						lowerCamelCase							(_SCREAMING_SNAKE_CASE   :     int     ):
  					__a    : Any      					=						datasets.load_dataset(args.dataset_name		,   args.dataset_config		,   split='train'     )
  
  					if args.limit is not None:
  										__a    : Union[str, Any]      					=						min(len(_SCREAMING_SNAKE_CASE     )		,   args.limit     )
  										__a    : int      					=						dataset.select(range(_SCREAMING_SNAKE_CASE     )     )
  										logger.info(F"""Limiting the dataset to {args.limit} entries."""     )
  
  					def batch_iterator():
  										for i in range(0		,   len(_SCREAMING_SNAKE_CASE     )		,   args.batch_size     ):
  															yield dataset[i : i + args.batch_size]["text"]
  
      # Prepare the tokenizer.
  					__a    : List[Any]      					=						Tokenizer(Unigram()     )
  					__a    : Union[str, Any]      					=						normalizers.Sequence([normalizers.Replace('``'		,   '"'     ), normalizers.Replace('\'\''		,   '"'     )]     )
  					__a    : Optional[int]      					=						pre_tokenizers.Metaspace()
  
  					# Prepare the trainer.
  					__a    : Dict      					=						UnigramTrainer(
  					    unk_token='<unk>'		,   special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']		,   vocab_size=args.vocab_size		,   )
  
  					logger.info('Training the tokenizer.'     )
  					tokenizer.train_from_iterator(batch_iterator()		,   trainer=_SCREAMING_SNAKE_CASE     )
  					logger.info('Tokenizer training complete!'     )
  
  					__a    : Optional[int]      					=						tokenizer.token_to_id('[CLS]'     )
  					__a    : int      					=						tokenizer.token_to_id('[SEP]'     )
  					__a    : List[Any]      					=						processors.TemplateProcessing(
  					    single='[CLS]:0 $A:0 [SEP]:0'		,   pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'		,   special_tokens=[
  					        ('[CLS]', cls_token_id),
  					        ('[SEP]', sep_token_id),
  					    ]		,   )
  					__a    : int      					=						decoders.Metaspace()
  
  					if args.export_to_hub:
  										logger.info('Exporting the trained tokenzier to Hub.'     )
  										__a    : Tuple      					=						AlbertTokenizerFast(tokenizer_object=_SCREAMING_SNAKE_CASE     )
  										new_tokenizer.push_to_hub('unigram-tokenizer-dataset'     )
  
  
  if __name__ == "__main__":
  	__lowercase			:		Dict =			parse_args()
  	main(args)
  
  
  
  
style_context: 
  
  
  '''simple docstring'''
  
  
  
  def 						lowerCamelCase							(_SCREAMING_SNAKE_CASE   :     set		,   _SCREAMING_SNAKE_CASE   :     set     ):
  					assert isinstance(_SCREAMING_SNAKE_CASE		,   _SCREAMING_SNAKE_CASE     ), F"""The input value of [set_a={set_a}] is not a set"""
  					assert isinstance(_SCREAMING_SNAKE_CASE		,   _SCREAMING_SNAKE_CASE     ), F"""The input value of [set_b={set_b}] is not a set"""
  
  					return {a + b for a in set_a for b in set_b}
  
  
  if __name__ == "__main__":
  	from doctest import testmod
  
  	testmod()
  
  
  
  