code: 
  
  
  """simple docstring"""
  
  
  
  
  import argparse
  import logging
  
  import datasets
  from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
  from tokenizers.models import Unigram
  from tokenizers.trainers import UnigramTrainer
  
  from transformers import AlbertTokenizerFast
  
  
  __SCREAMING_SNAKE_CASE					=						logging.getLogger(__name__)
  
  
  
  def 						__a      (  )     ->   Optional[Any]:
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         _a          =				argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."					)
         parser.add_argument(
             "--dataset_name",		type=a,		default="wikitext",		help="Name of the training. Explore datasets at: hf.co/datasets.",		)
         parser.add_argument(
             "--dataset_config",		type=a,		default="wikitext-103-raw-v1",		help="Configuration name of the dataset."					)
         parser.add_argument(
             "--batch_size",		type=a,		default=1_0_0_0,		help="Batch size during training.",		)
         parser.add_argument(
             "--vocab_size",		type=a,		default=1_0_0_4_8,		help="Size of the desired vocabulary.",		)
         parser.add_argument(
             "--limit",		default=a,		type=a,		help="Limit the number of shards (used for debugging).",		)
         parser.add_argument(
             "--export_to_hub",		action="store_true",		)
  
         _a          =				parser.parse_args()
         return args
  
  
  
  def 						__a      (  a					)     ->   List[Any]:
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         _a          =				datasets.load_dataset(args.dataset_name,		args.dataset_config,		split="train"					)
  
         if args.limit is not None:
                _a          =				min(len(a					),		args.limit					)
                _a          =				dataset.select(range(a					)					)
                logger.info(F'Limiting the dataset to {args.limit} entries.'					)
  
         def batch_iterator():
                for i in range(0,		len(a					),		args.batch_size					):
                       yield dataset[i : i + args.batch_size]["text"]
  
      # Prepare the tokenizer.
         _a          =				Tokenizer(Unigram()					)
         _a          =				normalizers.Sequence([normalizers.Replace("``",		"\""					), normalizers.Replace("''",		"\""					)]					)
         _a          =				pre_tokenizers.Metaspace()
  
         # Prepare the trainer.
         _a          =				UnigramTrainer(
             unk_token="<unk>",		special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"],		vocab_size=args.vocab_size,		)
  
         logger.info("Training the tokenizer."					)
         tokenizer.train_from_iterator(batch_iterator(),		trainer=a					)
         logger.info("Tokenizer training complete!"					)
  
         _a          =				tokenizer.token_to_id("[CLS]"					)
         _a          =				tokenizer.token_to_id("[SEP]"					)
         _a          =				processors.TemplateProcessing(
             single="[CLS]:0 $A:0 [SEP]:0",		pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1",		special_tokens=[
                 ("[CLS]", cls_token_id),
                 ("[SEP]", sep_token_id),
             ],		)
         _a          =				decoders.Metaspace()
  
         if args.export_to_hub:
                logger.info("Exporting the trained tokenzier to Hub."					)
                _a          =				AlbertTokenizerFast(tokenizer_object=a					)
                new_tokenizer.push_to_hub("unigram-tokenizer-dataset"					)
  
  
  if __name__ == "__main__":
     __SCREAMING_SNAKE_CASE					=						parse_args()
     main(args)
  
style_context: 
  
  
  """simple docstring"""
  
  
  
  
  import warnings
  
  from ...utils import logging
  from .image_processing_vit import ViTImageProcessor
  
  
  __SCREAMING_SNAKE_CASE					=						logging.get_logger(__name__)
  
  
  
  
  
  class __snake_case		(					_SCREAMING_SNAKE_CASE      ):
  
  
  
  
  
  
  
     """simple docstring"""
  
  
  
  
     def __init__(		self							:Tuple				,  *UpperCamelCase__							:Tuple				,  **UpperCamelCase__							:Any       ):
            warnings.warn(
                "The class ViTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"
                " use ViTImageProcessor instead."				,  UpperCamelCase__				,  )
            super().__init__(*UpperCamelCase__				,  **UpperCamelCase__       )
  