code: 
  
  
  
  
  
  
  from collections import OrderedDict
  from typing import Mapping
  
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfig
  from ...utils import logging
  
  
  lowerCamelCase			:							List[Any]       =logging.get_logger(__name__)
  
  lowerCamelCase			:							Optional[Any]       ={
      '''nghuyong/ernie-1.0-base-zh''': '''https://huggingface.co/nghuyong/ernie-1.0-base-zh/resolve/main/config.json''',
      '''nghuyong/ernie-2.0-base-en''': '''https://huggingface.co/nghuyong/ernie-2.0-base-en/resolve/main/config.json''',
      '''nghuyong/ernie-2.0-large-en''': '''https://huggingface.co/nghuyong/ernie-2.0-large-en/resolve/main/config.json''',
      '''nghuyong/ernie-3.0-base-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-base-zh/resolve/main/config.json''',
      '''nghuyong/ernie-3.0-medium-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-medium-zh/resolve/main/config.json''',
      '''nghuyong/ernie-3.0-mini-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-mini-zh/resolve/main/config.json''',
      '''nghuyong/ernie-3.0-micro-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-micro-zh/resolve/main/config.json''',
      '''nghuyong/ernie-3.0-nano-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-nano-zh/resolve/main/config.json''',
      '''nghuyong/ernie-gram-zh''': '''https://huggingface.co/nghuyong/ernie-gram-zh/resolve/main/config.json''',
      '''nghuyong/ernie-health-zh''': '''https://huggingface.co/nghuyong/ernie-health-zh/resolve/main/config.json''',
  }
  
  
  
  
  
  class     __a						(					A__ ):
   _lowerCAmelCase : int 				=				'''ernie'''
  
  
  
  
  
  
  
   def __init__(	self      :     Any							,     SCREAMING_SNAKE_CASE      :     Optional[Any]=3_05_22							,     SCREAMING_SNAKE_CASE      :     Dict=7_68							,     SCREAMING_SNAKE_CASE      :     int=12							,     SCREAMING_SNAKE_CASE      :     Tuple=12							,     SCREAMING_SNAKE_CASE      :     Optional[int]=30_72							,     SCREAMING_SNAKE_CASE      :     int="gelu"							,     SCREAMING_SNAKE_CASE      :     Union[str, Any]=0.1							,     SCREAMING_SNAKE_CASE      :     List[Any]=0.1							,     SCREAMING_SNAKE_CASE      :     Union[str, Any]=5_12							,     SCREAMING_SNAKE_CASE      :     Optional[Any]=2							,     SCREAMING_SNAKE_CASE      :     Union[str, Any]=3							,     SCREAMING_SNAKE_CASE      :     List[Any]=False							,     SCREAMING_SNAKE_CASE      :     Union[str, Any]=0.0_2							,     SCREAMING_SNAKE_CASE      :     int=1e-1_2							,     SCREAMING_SNAKE_CASE      :     List[Any]=0							,     SCREAMING_SNAKE_CASE      :     Any="absolute"							,     SCREAMING_SNAKE_CASE      :     List[Any]=True							,     SCREAMING_SNAKE_CASE      :     Dict=None							,     **SCREAMING_SNAKE_CASE      :     Any							,     ):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        super().__init__(pad_token_id=SCREAMING_SNAKE_CASE							,     **SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     Optional[Any]        =						vocab_size
        UpperCamelCase__    :     Optional[Any]        =						hidden_size
        UpperCamelCase__    :     List[Any]        =						num_hidden_layers
        UpperCamelCase__    :     Tuple        =						num_attention_heads
        UpperCamelCase__    :     List[str]        =						hidden_act
        UpperCamelCase__    :     Tuple        =						intermediate_size
        UpperCamelCase__    :     List[str]        =						hidden_dropout_prob
        UpperCamelCase__    :     Optional[Any]        =						attention_probs_dropout_prob
        UpperCamelCase__    :     List[str]        =						max_position_embeddings
        UpperCamelCase__    :     Union[str, Any]        =						type_vocab_size
        UpperCamelCase__    :     List[Any]        =						task_type_vocab_size
        UpperCamelCase__    :     Dict        =						use_task_id
        UpperCamelCase__    :     str        =						initializer_range
        UpperCamelCase__    :     Tuple        =						layer_norm_eps
        UpperCamelCase__    :     List[str]        =						position_embedding_type
        UpperCamelCase__    :     Optional[int]        =						use_cache
        UpperCamelCase__    :     List[Any]        =						classifier_dropout
  
  
  
  
  
  
  
  class     __a						(					A__ ):
  
  
  
  
  
  
  
   @property
   def       __lowercase      (	self      :     Any		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        if self.task == "multiple-choice":
             UpperCamelCase__    :     int        =						{0: "batch", 1: "choice", 2: "sequence"}
        else:
             UpperCamelCase__    :     Optional[int]        =						{0: "batch", 1: "sequence"}
        return OrderedDict(
            [
                ("input_ids", dynamic_axis),
                ("attention_mask", dynamic_axis),
                ("token_type_ids", dynamic_axis),
                ("task_type_ids", dynamic_axis),
            ]		)
style_context: 
  
  
  
  
  
  
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
   print('''#################### AVAILABLE DEVICES ####################''')
   print(jax.devices())
   print('''###########################################################''')
  
   # setup for wandb sweep
   lowerCamelCase			:							str       =Args()
   lowerCamelCase			:							str       =wandb.init(project='''bigbird-natural-questions''', config=args.__dict__)
   lowerCamelCase			:							Any       =dict(logger.config)
   del wandb_args["batch_size"]
   lowerCamelCase			:							str       =replace(args, **wandb_args)
   lowerCamelCase			:							List[str]       =args.base_dir + '''-''' + wandb.run.id
   lowerCamelCase			:							str       =replace(args, base_dir=base_dir)
   print(args)
  
   lowerCamelCase			:							List[Any]       =load_dataset('''json''', data_files=args.tr_data_path)['''train''']
   lowerCamelCase			:							str       =load_dataset('''json''', data_files=args.val_data_path)['''train''']
  
   # drop extra batch for now
   lowerCamelCase			:							List[Any]       =range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
   lowerCamelCase			:							Union[str, Any]       =tr_dataset.shuffle().select(indices)
   lowerCamelCase			:							List[str]       =range(len(val_dataset) - len(val_dataset) % args.batch_size)
   lowerCamelCase			:							Dict       =val_dataset.shuffle().select(indices)
  
   if os.environ.get('''TRAIN_ON_SMALL''', '''false''') == "true":
    lowerCamelCase			:							Optional[int]       =tr_dataset.shuffle().select(range(80000))
    lowerCamelCase			:							List[Any]       =val_dataset.shuffle().select(range(8000))
  
   print(tr_dataset)
   print(val_dataset)
  
   lowerCamelCase			:							int       =FlaxBigBirdForNaturalQuestions.from_pretrained(
       args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
   )
   lowerCamelCase			:							Union[str, Any]       =BigBirdTokenizerFast.from_pretrained(args.model_id)
   lowerCamelCase			:							Dict       =DataCollator(pad_id=tokenizer.pad_token_id, max_length=4096)
  
   lowerCamelCase			:							List[str]       ={
       '''lr''': args.lr,
       '''init_lr''': args.init_lr,
       '''warmup_steps''': args.warmup_steps,
       '''num_train_steps''': args.max_epochs * (len(tr_dataset) // args.batch_size),
       '''weight_decay''': args.weight_decay,
   }
   lowerCamelCase       ,    lowerCamelCase			:							Tuple       =build_tx(**tx_args)
  
   lowerCamelCase			:							Optional[Any]       =Trainer(
       args=args,
       data_collator=data_collator,
       model_save_fn=model.save_pretrained,
       train_step_fn=train_step,
       val_step_fn=val_step,
       logger=logger,
       scheduler_fn=lr,
   )
  
   lowerCamelCase			:							Tuple       =None
   lowerCamelCase			:							str       =trainer.create_state(model, tx, num_train_steps=tx_args['''num_train_steps'''], ckpt_dir=ckpt_dir)
   try:
    trainer.train(state, tr_dataset, val_dataset)
   except KeyboardInterrupt:
    print('''Oooops; TRAINING STOPPED UNFORTUNATELY''')
  
   print('''SAVING WEIGHTS IN `final-weights`''')
   lowerCamelCase			:							str       =jax_utils.unreplicate(state.params)
   model.save_pretrained(os.path.join(args.base_dir, '''final-weights'''), params=params)