code: 
  
  
  
  
  
  
  import argparse
  import logging
  
  import datasets
  from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
  from tokenizers.models import Unigram
  from tokenizers.trainers import UnigramTrainer
  
  from transformers import AlbertTokenizerFast
  
  
  SCREAMING_SNAKE_CASE__		:       Union[str, Any]   							=							logging.getLogger(__name__)
  
  
  
  
  
  def    _lowerCamelCase	(	)   -> Optional[Any]:
  
       '''simple docstring'''
       UpperCAmelCase__			:     Any										= argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""					)
       parser.add_argument(
           """--dataset_name"""   ,       type=__lowerCamelCase   ,       default="""wikitext"""   ,       help="""Name of the training. Explore datasets at: hf.co/datasets."""   ,       )
       parser.add_argument(
           """--dataset_config"""   ,       type=__lowerCamelCase   ,       default="""wikitext-103-raw-v1"""   ,       help="""Configuration name of the dataset."""					)
       parser.add_argument(
           """--batch_size"""   ,       type=__lowerCamelCase   ,       default=1000   ,       help="""Batch size during training."""   ,       )
       parser.add_argument(
           """--vocab_size"""   ,       type=__lowerCamelCase   ,       default=1_0048   ,       help="""Size of the desired vocabulary."""   ,       )
       parser.add_argument(
           """--limit"""   ,       default=__lowerCamelCase   ,       type=__lowerCamelCase   ,       help="""Limit the number of shards (used for debugging)."""   ,       )
       parser.add_argument(
           """--export_to_hub"""   ,       action="""store_true"""   ,       )
  
       UpperCAmelCase__			:     Tuple										= parser.parse_args()
       return args
  
  
  
  
  
  def    _lowerCamelCase	(	__lowerCamelCase					)   -> str:
  
       '''simple docstring'''
       UpperCAmelCase__			:     str										= datasets.load_dataset(args.dataset_name   ,       args.dataset_config   ,       split="""train"""					)
  
       if args.limit is not None:
            UpperCAmelCase__			:     Optional[int]										= min(len(__lowerCamelCase					)   ,       args.limit					)
            UpperCAmelCase__			:     List[Any]										= dataset.select(range(__lowerCamelCase					)					)
            logger.info(F"Limiting the dataset to {args.limit} entries."					)
  
       def batch_iterator():
            for i in range(0   ,       len(__lowerCamelCase					)   ,       args.batch_size					):
                 yield dataset[i : i + args.batch_size]["text"]
  
      # Prepare the tokenizer.
       UpperCAmelCase__			:     str										= Tokenizer(Unigram()					)
       UpperCAmelCase__			:     Any										= normalizers.Sequence([normalizers.Replace("""``"""   ,       """\""""					), normalizers.Replace("""''"""   ,       """\""""					)]					)
       UpperCAmelCase__			:     Union[str, Any]										= pre_tokenizers.Metaspace()
  
       # Prepare the trainer.
       UpperCAmelCase__			:     Optional[int]										= UnigramTrainer(
           unk_token="""<unk>"""   ,       special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]   ,       vocab_size=args.vocab_size   ,       )
  
       logger.info("""Training the tokenizer."""					)
       tokenizer.train_from_iterator(batch_iterator()   ,       trainer=__lowerCamelCase					)
       logger.info("""Tokenizer training complete!"""					)
  
       UpperCAmelCase__			:     List[str]										= tokenizer.token_to_id("""[CLS]"""					)
       UpperCAmelCase__			:     int										= tokenizer.token_to_id("""[SEP]"""					)
       UpperCAmelCase__			:     List[Any]										= processors.TemplateProcessing(
           single="""[CLS]:0 $A:0 [SEP]:0"""   ,       pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""   ,       special_tokens=[
               ("""[CLS]""", cls_token_id),
               ("""[SEP]""", sep_token_id),
           ]   ,       )
       UpperCAmelCase__			:     str										= decoders.Metaspace()
  
       if args.export_to_hub:
            logger.info("""Exporting the trained tokenzier to Hub."""					)
            UpperCAmelCase__			:     Dict										= AlbertTokenizerFast(tokenizer_object=__lowerCamelCase					)
            new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""					)
  
  
  if __name__ == "__main__":
      SCREAMING_SNAKE_CASE__		:       Any   							=							parse_args()
      main(args)
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  from ..utils import requires_backends
  
  
  SCREAMING_SNAKE_CASE__		:       int   							=							None
  
  
  
  
  
  
  
  class 							UpperCAmelCase_     :
  
  
  
  
       def __init__(							self  ,	*_lowerCAmelCase  ,	**_lowerCAmelCase		):
            requires_backends(self  ,	["""detectron2"""]		)
  
  
  
  
       @classmethod
       def      __UpperCAmelCase     (							cls  ,	*_lowerCAmelCase  ,	**_lowerCAmelCase		):
            requires_backends(cls  ,	["""detectron2"""]		)
  
  
  
  
  
  
  