code: 
  
  
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
  	print("#################### AVAILABLE DEVICES ####################")
  	print(jax.devices())
  	print("###########################################################")
  
  	# setup for wandb sweep
  	SCREAMING_SNAKE_CASE : int          =    Args()
  	SCREAMING_SNAKE_CASE : Any          =    wandb.init(project="bigbird-natural-questions", config=args.__dict__)
  	SCREAMING_SNAKE_CASE : Tuple          =    dict(logger.config)
  	del wandb_args["batch_size"]
  	SCREAMING_SNAKE_CASE : Dict          =    replace(args, **wandb_args)
  	SCREAMING_SNAKE_CASE : str          =    args.base_dir + "-" + wandb.run.id
  	SCREAMING_SNAKE_CASE : Any          =    replace(args, base_dir=base_dir)
  	print(args)
  
  	SCREAMING_SNAKE_CASE : List[Any]          =    load_dataset("json", data_files=args.tr_data_path)["train"]
  	SCREAMING_SNAKE_CASE : Dict          =    load_dataset("json", data_files=args.val_data_path)["train"]
  
  	# drop extra batch for now
  	SCREAMING_SNAKE_CASE : int          =    range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
  	SCREAMING_SNAKE_CASE : Optional[int]          =    tr_dataset.shuffle().select(indices)
  	SCREAMING_SNAKE_CASE : str          =    range(len(val_dataset) - len(val_dataset) % args.batch_size)
  	SCREAMING_SNAKE_CASE : Optional[int]          =    val_dataset.shuffle().select(indices)
  
  	if os.environ.get("TRAIN_ON_SMALL", "false") == "true":
  		SCREAMING_SNAKE_CASE : Tuple          =    tr_dataset.shuffle().select(range(8_00_00))
  		SCREAMING_SNAKE_CASE : Tuple          =    val_dataset.shuffle().select(range(80_00))
  
  	print(tr_dataset)
  	print(val_dataset)
  
  	SCREAMING_SNAKE_CASE : List[Any]          =    FlaxBigBirdForNaturalQuestions.from_pretrained(
  	    args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
  	)
  	SCREAMING_SNAKE_CASE : Any          =    BigBirdTokenizerFast.from_pretrained(args.model_id)
  	SCREAMING_SNAKE_CASE : Any          =    DataCollator(pad_id=tokenizer.pad_token_id, max_length=40_96)
  
  	SCREAMING_SNAKE_CASE : Optional[int]          =    {
  	    "lr": args.lr,
  	    "init_lr": args.init_lr,
  	    "warmup_steps": args.warmup_steps,
  	    "num_train_steps": args.max_epochs * (len(tr_dataset) // args.batch_size),
  	    "weight_decay": args.weight_decay,
  	}
  	SCREAMING_SNAKE_CASE    ,						SCREAMING_SNAKE_CASE : Dict          =    build_tx(**tx_args)
  
  	SCREAMING_SNAKE_CASE : Union[str, Any]          =    Trainer(
  	    args=args,
  	    data_collator=data_collator,
  	    model_save_fn=model.save_pretrained,
  	    train_step_fn=train_step,
  	    val_step_fn=val_step,
  	    logger=logger,
  	    scheduler_fn=lr,
  	)
  
  	SCREAMING_SNAKE_CASE : int          =    None
  	SCREAMING_SNAKE_CASE : Tuple          =    trainer.create_state(model, tx, num_train_steps=tx_args["num_train_steps"], ckpt_dir=ckpt_dir)
  	try:
  		trainer.train(state, tr_dataset, val_dataset)
  	except KeyboardInterrupt:
  		print("Oooops; TRAINING STOPPED UNFORTUNATELY")
  
  	print("SAVING WEIGHTS IN `final-weights`")
  	SCREAMING_SNAKE_CASE : Tuple          =    jax_utils.unreplicate(state.params)
  	model.save_pretrained(os.path.join(args.base_dir, "final-weights"), params=params)
  
  
  
style_context: 
  
  
  from collections import OrderedDict
  from typing import Mapping
  
  from packaging import version
  
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfig
  from ...utils import logging
  
  
  SCREAMING_SNAKE_CASE : Dict          =    logging.get_logger(__name__)
  
  SCREAMING_SNAKE_CASE : str          =    {
      "apple/mobilevit-small": "https://huggingface.co/apple/mobilevit-small/resolve/main/config.json",
      "apple/mobilevit-x-small": "https://huggingface.co/apple/mobilevit-x-small/resolve/main/config.json",
      "apple/mobilevit-xx-small": "https://huggingface.co/apple/mobilevit-xx-small/resolve/main/config.json",
      "apple/deeplabv3-mobilevit-small": (
          "https://huggingface.co/apple/deeplabv3-mobilevit-small/resolve/main/config.json"
      ),
      "apple/deeplabv3-mobilevit-x-small": (
          "https://huggingface.co/apple/deeplabv3-mobilevit-x-small/resolve/main/config.json"
      ),
      "apple/deeplabv3-mobilevit-xx-small": (
          "https://huggingface.co/apple/deeplabv3-mobilevit-xx-small/resolve/main/config.json"
      ),
      # See all MobileViT models at https://huggingface.co/models?filter=mobilevit
  }
  
  
  
  
  
  class 					snake_case__  ( __A       ):
  	UpperCAmelCase   :							List[str]         = """mobilevit"""
  
  
  
  
  
  	def __init__(  self				,    UpperCamelCase_=3				,    UpperCamelCase_=256				,    UpperCamelCase_=2				,    UpperCamelCase_=[144, 192, 240]				,    UpperCamelCase_=[16, 32, 64, 96, 128, 160, 640]				,    UpperCamelCase_=4				,    UpperCamelCase_=2.0				,    UpperCamelCase_=4.0				,    UpperCamelCase_="silu"				,    UpperCamelCase_=3				,    UpperCamelCase_=32				,    UpperCamelCase_=0.1				,    UpperCamelCase_=0.0				,    UpperCamelCase_=0.1				,    UpperCamelCase_=0.02				,    UpperCamelCase_=1e-5				,    UpperCamelCase_=True				,    UpperCamelCase_=256				,    UpperCamelCase_=[6, 12, 18]				,    UpperCamelCase_=0.1				,    UpperCamelCase_=255				,    **UpperCamelCase_				,    )		->			str:
  								"""simple docstring"""
  
  
  
  
  
  
  								super().__init__(**UpperCamelCase_							)
  
  								a_       :   Optional[int]			=	num_channels
  								a_       :   Optional[Any]			=	image_size
  								a_       :   int			=	patch_size
  								a_       :   List[str]			=	hidden_sizes
  								a_       :   Optional[int]			=	neck_hidden_sizes
  								a_       :   List[Any]			=	num_attention_heads
  								a_       :   Union[str, Any]			=	mlp_ratio
  								a_       :   List[str]			=	expand_ratio
  								a_       :   str			=	hidden_act
  								a_       :   Union[str, Any]			=	conv_kernel_size
  								a_       :   Optional[Any]			=	output_stride
  								a_       :   Union[str, Any]			=	hidden_dropout_prob
  								a_       :   List[Any]			=	attention_probs_dropout_prob
  								a_       :   List[Any]			=	classifier_dropout_prob
  								a_       :   Union[str, Any]			=	initializer_range
  								a_       :   List[Any]			=	layer_norm_eps
  								a_       :   str			=	qkv_bias
  
  								# decode head attributes for semantic segmentation
  								a_       :   Optional[Any]			=	aspp_out_channels
  								a_       :   Any			=	atrous_rates
  								a_       :   Union[str, Any]			=	aspp_dropout_prob
  								a_       :   Any			=	semantic_loss_ignore_index
  
  
  
  
  
  class 					snake_case__  ( __A       ):
  	UpperCAmelCase   :							Optional[int]         = version.parse("""1.11"""       )
  
  
  
  
  
  	@property
  	def     A				(  self							)		->			Mapping[str, Mapping[int, str]]:
  								"""simple docstring"""
  
  
  
  
  
  
  								return OrderedDict([("""pixel_values""", {0: """batch""", 1: """num_channels""", 2: """height""", 3: """width"""})]							)
  
  
  
  
  
  	@property
  	def     A				(  self							)		->			Mapping[str, Mapping[int, str]]:
  								"""simple docstring"""
  
  
  
  
  
  
  								if self.task == "image-classification":
  															return OrderedDict([("""logits""", {0: """batch"""})]							)
  								else:
  															return OrderedDict([("""last_hidden_state""", {0: """batch"""}), ("""pooler_output""", {0: """batch"""})]							)
  
  
  
  
  
  	@property
  	def     A				(  self							)		->			float:
  								"""simple docstring"""
  
  
  
  
  
  
  								return 1e-4
  
  
  