code: 
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
  				print("#################### AVAILABLE DEVICES ####################")
  				print(jax.devices())
  				print("###########################################################")
  
  				# setup for wandb sweep
  				snake_case				:	Dict							   =						Args()
  				snake_case				:	str							   =						wandb.init(project="bigbird-natural-questions", config=args.__dict__)
  				snake_case				:	Dict							   =						dict(logger.config)
  				del wandb_args["batch_size"]
  				snake_case				:	Optional[Any]							   =						replace(args, **wandb_args)
  				snake_case				:	List[Any]							   =						args.base_dir + "-" + wandb.run.id
  				snake_case				:	Union[str, Any]							   =						replace(args, base_dir=base_dir)
  				print(args)
  
  				snake_case				:	int							   =						load_dataset("json", data_files=args.tr_data_path)["train"]
  				snake_case				:	Optional[Any]							   =						load_dataset("json", data_files=args.val_data_path)["train"]
  
  				# drop extra batch for now
  				snake_case				:	int							   =						range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
  				snake_case				:	Union[str, Any]							   =						tr_dataset.shuffle().select(indices)
  				snake_case				:	Dict							   =						range(len(val_dataset) - len(val_dataset) % args.batch_size)
  				snake_case				:	Any							   =						val_dataset.shuffle().select(indices)
  
  				if os.environ.get("TRAIN_ON_SMALL", "false") == "true":
  								snake_case				:	Tuple							   =						tr_dataset.shuffle().select(range(80_000))
  								snake_case				:	List[str]							   =						val_dataset.shuffle().select(range(8_000))
  
  				print(tr_dataset)
  				print(val_dataset)
  
  				snake_case				:	Tuple							   =						FlaxBigBirdForNaturalQuestions.from_pretrained(
  				    args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
  				)
  				snake_case				:	Dict							   =						BigBirdTokenizerFast.from_pretrained(args.model_id)
  				snake_case				:	Any							   =						DataCollator(pad_id=tokenizer.pad_token_id, max_length=4_096)
  
  				snake_case				:	int							   =						{
  				    "lr": args.lr,
  				    "init_lr": args.init_lr,
  				    "warmup_steps": args.warmup_steps,
  				    "num_train_steps": args.max_epochs * (len(tr_dataset) // args.batch_size),
  				    "weight_decay": args.weight_decay,
  				}
  				snake_case   ,snake_case				:	Dict							   =						build_tx(**tx_args)
  
  				snake_case				:	Union[str, Any]							   =						Trainer(
  				    args=args,
  				    data_collator=data_collator,
  				    model_save_fn=model.save_pretrained,
  				    train_step_fn=train_step,
  				    val_step_fn=val_step,
  				    logger=logger,
  				    scheduler_fn=lr,
  				)
  
  				snake_case				:	List[str]							   =						None
  				snake_case				:	Tuple							   =						trainer.create_state(model, tx, num_train_steps=tx_args["num_train_steps"], ckpt_dir=ckpt_dir)
  				try:
  								trainer.train(state, tr_dataset, val_dataset)
  				except KeyboardInterrupt:
  								print("Oooops; TRAINING STOPPED UNFORTUNATELY")
  
  				print("SAVING WEIGHTS IN `final-weights`")
  				snake_case				:	Union[str, Any]							   =						jax_utils.unreplicate(state.params)
  				model.save_pretrained(os.path.join(args.base_dir, "final-weights"), params=params)
  
style_context: 
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_torch_available,
      is_vision_available,
  )
  
  
  snake_case				:	Optional[int]							   =						{
      "configuration_mobilevitv2": [
          "MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP",
          "MobileViTV2Config",
          "MobileViTV2OnnxConfig",
      ],
  }
  
  
  try:
  				if not is_torch_available():
  								raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  				pass
  else:
  				snake_case				:	Optional[int]							   =						[
  				    "MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST",
  				    "MobileViTV2ForImageClassification",
  				    "MobileViTV2ForSemanticSegmentation",
  				    "MobileViTV2Model",
  				    "MobileViTV2PreTrainedModel",
  				]
  
  if TYPE_CHECKING:
  				from .configuration_mobilevitva import (
  				    MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP,
  				    MobileViTVaConfig,
  				    MobileViTVaOnnxConfig,
  				)
  
  				try:
  								if not is_torch_available():
  												raise OptionalDependencyNotAvailable()
  				except OptionalDependencyNotAvailable:
  								pass
  				else:
  								from .modeling_mobilevitva import (
  								    MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST,
  								    MobileViTVaForImageClassification,
  								    MobileViTVaForSemanticSegmentation,
  								    MobileViTVaModel,
  								    MobileViTVaPreTrainedModel,
  								)
  
  else:
  				import sys
  
  				snake_case				:	Tuple							   =						_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
  