code: 
  
  import argparse
  import logging
  
  import datasets
  from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
  from tokenizers.models import Unigram
  from tokenizers.trainers import UnigramTrainer
  
  from transformers import AlbertTokenizerFast
  
  
  snake_case_    						=				logging.getLogger(__name__)
  
  def 			snake_case__    (					):
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  					lowercase__					:     Any   = argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'    )
  					parser.add_argument(
  					    '--dataset_name'			,  type=SCREAMING_SNAKE_CASE_			,  default='wikitext'			,  help='Name of the training. Explore datasets at: hf.co/datasets.'			,  )
  					parser.add_argument(
  					    '--dataset_config'			,  type=SCREAMING_SNAKE_CASE_			,  default='wikitext-103-raw-v1'			,  help='Configuration name of the dataset.'    )
  					parser.add_argument(
  					    '--batch_size'			,  type=SCREAMING_SNAKE_CASE_			,  default=1_000			,  help='Batch size during training.'			,  )
  					parser.add_argument(
  					    '--vocab_size'			,  type=SCREAMING_SNAKE_CASE_			,  default=10_048			,  help='Size of the desired vocabulary.'			,  )
  					parser.add_argument(
  					    '--limit'			,  default=SCREAMING_SNAKE_CASE_			,  type=SCREAMING_SNAKE_CASE_			,  help='Limit the number of shards (used for debugging).'			,  )
  					parser.add_argument(
  					    '--export_to_hub'			,  action='store_true'			,  )
  
  					lowercase__					:     List[Any]   = parser.parse_args()
  					return args
  
  def 			snake_case__    (					SCREAMING_SNAKE_CASE_	:					int    ):
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  					lowercase__					:     List[str]   = datasets.load_dataset(args.dataset_name			,  args.dataset_config			,  split='train'    )
  
  					if args.limit is not None:
  										lowercase__					:     List[str]   = min(len(SCREAMING_SNAKE_CASE_    )			,  args.limit    )
  										lowercase__					:     Dict   = dataset.select(range(SCREAMING_SNAKE_CASE_    )    )
  										logger.info(f"""Limiting the dataset to {args.limit} entries."""    )
  
  					def batch_iterator():
  										for i in range(0			,  len(SCREAMING_SNAKE_CASE_    )			,  args.batch_size    ):
  															yield dataset[i : i + args.batch_size]["text"]
  
      # Prepare the tokenizer.
  					lowercase__					:     Tuple   = Tokenizer(Unigram()    )
  					lowercase__					:     Optional[Any]   = normalizers.Sequence([normalizers.Replace('``'			,  '"'    ), normalizers.Replace('\'\''			,  '"'    )]    )
  					lowercase__					:     List[str]   = pre_tokenizers.Metaspace()
  
  					# Prepare the trainer.
  					lowercase__					:     Dict   = UnigramTrainer(
  					    unk_token='<unk>'			,  special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']			,  vocab_size=args.vocab_size			,  )
  
  					logger.info('Training the tokenizer.'    )
  					tokenizer.train_from_iterator(batch_iterator()			,  trainer=SCREAMING_SNAKE_CASE_    )
  					logger.info('Tokenizer training complete!'    )
  
  					lowercase__					:     Union[str, Any]   = tokenizer.token_to_id('[CLS]'    )
  					lowercase__					:     Any   = tokenizer.token_to_id('[SEP]'    )
  					lowercase__					:     Any   = processors.TemplateProcessing(
  					    single='[CLS]:0 $A:0 [SEP]:0'			,  pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'			,  special_tokens=[
  					        ('[CLS]', cls_token_id),
  					        ('[SEP]', sep_token_id),
  					    ]			,  )
  					lowercase__					:     Any   = decoders.Metaspace()
  
  					if args.export_to_hub:
  										logger.info('Exporting the trained tokenzier to Hub.'    )
  										lowercase__					:     Any   = AlbertTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE_    )
  										new_tokenizer.push_to_hub('unigram-tokenizer-dataset'    )
  
  
  if __name__ == "__main__":
  							snake_case_    						=				parse_args()
  							main(args)
  
  
style_context: 
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_tf_available,
      is_torch_available,
  )
  
  
  snake_case_    						=				{'''configuration_gptj''': ['''GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''GPTJConfig''', '''GPTJOnnxConfig''']}
  
  try:
         if not is_torch_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         snake_case_    						=				[
             '''GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST''',
             '''GPTJForCausalLM''',
             '''GPTJForQuestionAnswering''',
             '''GPTJForSequenceClassification''',
             '''GPTJModel''',
             '''GPTJPreTrainedModel''',
         ]
  
  try:
         if not is_tf_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         snake_case_    						=				[
             '''TFGPTJForCausalLM''',
             '''TFGPTJForQuestionAnswering''',
             '''TFGPTJForSequenceClassification''',
             '''TFGPTJModel''',
             '''TFGPTJPreTrainedModel''',
         ]
  
  try:
         if not is_flax_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         snake_case_    						=				[
             '''FlaxGPTJForCausalLM''',
             '''FlaxGPTJModel''',
             '''FlaxGPTJPreTrainedModel''',
         ]
  
  
  if TYPE_CHECKING:
         from .configuration_gptj import GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTJConfig, GPTJOnnxConfig
  
         try:
                if not is_torch_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from .modeling_gptj import (
                    GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST,
                    GPTJForCausalLM,
                    GPTJForQuestionAnswering,
                    GPTJForSequenceClassification,
                    GPTJModel,
                    GPTJPreTrainedModel,
                )
  
         try:
                if not is_tf_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from .modeling_tf_gptj import (
                    TFGPTJForCausalLM,
                    TFGPTJForQuestionAnswering,
                    TFGPTJForSequenceClassification,
                    TFGPTJModel,
                    TFGPTJPreTrainedModel,
                )
  
         try:
                if not is_flax_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from .modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel, FlaxGPTJPreTrainedModel
  
  else:
         import sys
  
         snake_case_    						=				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
  
  