code: 
  
  
  
  
  
  
  import argparse
  import logging
  
  import datasets
  from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
  from tokenizers.models import Unigram
  from tokenizers.trainers import UnigramTrainer
  
  from transformers import AlbertTokenizerFast
  
  
  UpperCamelCase_	:    Union[str, Any]  					=       logging.getLogger(__name__)
  
  
  
  def 		UpperCamelCase    (					)      -> List[Any]:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						_lowercase					:  Dict    			=      argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."     )
  						parser.add_argument(
  						    "--dataset_name"   , type=_UpperCAmelCase   , default="wikitext"   , help="Name of the training. Explore datasets at: hf.co/datasets."   , )
  						parser.add_argument(
  						    "--dataset_config"   , type=_UpperCAmelCase   , default="wikitext-103-raw-v1"   , help="Configuration name of the dataset."     )
  						parser.add_argument(
  						    "--batch_size"   , type=_UpperCAmelCase   , default=1000   , help="Batch size during training."   , )
  						parser.add_argument(
  						    "--vocab_size"   , type=_UpperCAmelCase   , default=1_0048   , help="Size of the desired vocabulary."   , )
  						parser.add_argument(
  						    "--limit"   , default=_UpperCAmelCase   , type=_UpperCAmelCase   , help="Limit the number of shards (used for debugging)."   , )
  						parser.add_argument(
  						    "--export_to_hub"   , action="store_true"   , )
  
  						_lowercase					:  Optional[int]    			=      parser.parse_args()
  						return args
  
  
  
  def 		UpperCamelCase    (					_UpperCAmelCase     :      List[str]     )      -> Dict:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						_lowercase					:  Optional[int]    			=      datasets.load_dataset(args.dataset_name   , args.dataset_config   , split="train"     )
  
  						if args.limit is not None:
  												_lowercase					:  Union[str, Any]    			=      min(len(_UpperCAmelCase     )   , args.limit     )
  												_lowercase					:  Optional[int]    			=      dataset.select(range(_UpperCAmelCase     )     )
  												logger.info(f"""Limiting the dataset to {args.limit} entries."""     )
  
  						def batch_iterator():
  												for i in range(0   , len(_UpperCAmelCase     )   , args.batch_size     ):
  																		yield dataset[i : i + args.batch_size]["text"]
  
      # Prepare the tokenizer.
  						_lowercase					:  Optional[Any]    			=      Tokenizer(Unigram()     )
  						_lowercase					:  Optional[Any]    			=      normalizers.Sequence([normalizers.Replace("``"   , "\""     ), normalizers.Replace("''"   , "\""     )]     )
  						_lowercase					:  Optional[Any]    			=      pre_tokenizers.Metaspace()
  
  						# Prepare the trainer.
  						_lowercase					:  Tuple    			=      UnigramTrainer(
  						    unk_token="<unk>"   , special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]   , vocab_size=args.vocab_size   , )
  
  						logger.info("Training the tokenizer."     )
  						tokenizer.train_from_iterator(batch_iterator()   , trainer=_UpperCAmelCase     )
  						logger.info("Tokenizer training complete!"     )
  
  						_lowercase					:  Tuple    			=      tokenizer.token_to_id("[CLS]"     )
  						_lowercase					:  str    			=      tokenizer.token_to_id("[SEP]"     )
  						_lowercase					:  Dict    			=      processors.TemplateProcessing(
  						    single="[CLS]:0 $A:0 [SEP]:0"   , pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"   , special_tokens=[
  						        ("[CLS]", cls_token_id),
  						        ("[SEP]", sep_token_id),
  						    ]   , )
  						_lowercase					:  int    			=      decoders.Metaspace()
  
  						if args.export_to_hub:
  												logger.info("Exporting the trained tokenzier to Hub."     )
  												_lowercase					:  List[Any]    			=      AlbertTokenizerFast(tokenizer_object=_UpperCAmelCase     )
  												new_tokenizer.push_to_hub("unigram-tokenizer-dataset"     )
  
  
  if __name__ == "__main__":
  							UpperCamelCase_	:    Dict  					=       parse_args()
  							main(args)
  
  
  
  
  
style_context: 
  import math
  
  
  
  def 		UpperCamelCase    (					_UpperCAmelCase     :      int     )      -> list:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						if n <= 0:
  												raise ValueError("Only positive integers have prime factors"     )
  						_lowercase					:  Optional[int]    			=      []
  						while n % 2 == 0:
  												pf.append(2     )
  												_lowercase					:  Tuple    			=      int(n / 2     )
  						for i in range(3   , int(math.sqrt(_UpperCAmelCase     )     ) + 1   , 2     ):
  												while n % i == 0:
  																		pf.append(_UpperCAmelCase     )
  																		_lowercase					:  Optional[int]    			=      int(n / i     )
  						if n > 2:
  												pf.append(_UpperCAmelCase     )
  						return pf
  
  
  
  def 		UpperCamelCase    (					_UpperCAmelCase     :      int     )      -> int:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						if n <= 0:
  												raise ValueError("Only positive numbers are accepted"     )
  						_lowercase					:  str    			=      1
  						_lowercase					:  Tuple    			=      1
  						while n % 2 == 0:
  												temp += 1
  												_lowercase					:  List[Any]    			=      int(n / 2     )
  						div *= temp
  						for i in range(3   , int(math.sqrt(_UpperCAmelCase     )     ) + 1   , 2     ):
  												_lowercase					:  Optional[Any]    			=      1
  												while n % i == 0:
  																		temp += 1
  																		_lowercase					:  Optional[Any]    			=      int(n / i     )
  												div *= temp
  						if n > 1:
  												div *= 2
  						return div
  
  
  
  def 		UpperCamelCase    (					_UpperCAmelCase     :      int     )      -> int:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						if n <= 0:
  												raise ValueError("Only positive numbers are accepted"     )
  						_lowercase					:  int    			=      1
  						_lowercase					:  List[str]    			=      1
  						while n % 2 == 0:
  												temp += 1
  												_lowercase					:  str    			=      int(n / 2     )
  						if temp > 1:
  												s *= (2**temp - 1) / (2 - 1)
  						for i in range(3   , int(math.sqrt(_UpperCAmelCase     )     ) + 1   , 2     ):
  												_lowercase					:  int    			=      1
  												while n % i == 0:
  																		temp += 1
  																		_lowercase					:  List[Any]    			=      int(n / i     )
  												if temp > 1:
  																		s *= (i**temp - 1) / (i - 1)
  						return int(_UpperCAmelCase     )
  
  
  
  def 		UpperCamelCase    (					_UpperCAmelCase     :      int     )      -> int:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						_lowercase					:  int    			=      n
  						for x in set(prime_factors(_UpperCAmelCase     )     ):
  												s *= (x - 1) / x
  						return int(_UpperCAmelCase     )
  
  
  if __name__ == "__main__":
  							import doctest
  
  							doctest.testmod()
  
  
  
  
  