code: 
  
  
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
  		print('#################### AVAILABLE DEVICES ####################')
  		print(jax.devices())
  		print('###########################################################')
  
  		# setup for wandb sweep
  		lowercase_								=    Args()
  		lowercase_								=    wandb.init(project='bigbird-natural-questions', config=args.__dict__)
  		lowercase_								=    dict(logger.config)
  		del wandb_args["batch_size"]
  		lowercase_								=    replace(args, **wandb_args)
  		lowercase_								=    args.base_dir + '-' + wandb.run.id
  		lowercase_								=    replace(args, base_dir=base_dir)
  		print(args)
  
  		lowercase_								=    load_dataset('json', data_files=args.tr_data_path)['train']
  		lowercase_								=    load_dataset('json', data_files=args.val_data_path)['train']
  
  		# drop extra batch for now
  		lowercase_								=    range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
  		lowercase_								=    tr_dataset.shuffle().select(indices)
  		lowercase_								=    range(len(val_dataset) - len(val_dataset) % args.batch_size)
  		lowercase_								=    val_dataset.shuffle().select(indices)
  
  		if os.environ.get('TRAIN_ON_SMALL', 'false') == "true":
  				lowercase_								=    tr_dataset.shuffle().select(range(8_00_00))
  				lowercase_								=    val_dataset.shuffle().select(range(80_00))
  
  		print(tr_dataset)
  		print(val_dataset)
  
  		lowercase_								=    FlaxBigBirdForNaturalQuestions.from_pretrained(
  		    args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
  		)
  		lowercase_								=    BigBirdTokenizerFast.from_pretrained(args.model_id)
  		lowercase_								=    DataCollator(pad_id=tokenizer.pad_token_id, max_length=40_96)
  
  		lowercase_								=    {
  		    'lr': args.lr,
  		    'init_lr': args.init_lr,
  		    'warmup_steps': args.warmup_steps,
  		    'num_train_steps': args.max_epochs * (len(tr_dataset) // args.batch_size),
  		    'weight_decay': args.weight_decay,
  		}
  		lowercase_		,      lowercase_								=    build_tx(**tx_args)
  
  		lowercase_								=    Trainer(
  		    args=args,
  		    data_collator=data_collator,
  		    model_save_fn=model.save_pretrained,
  		    train_step_fn=train_step,
  		    val_step_fn=val_step,
  		    logger=logger,
  		    scheduler_fn=lr,
  		)
  
  		lowercase_								=    None
  		lowercase_								=    trainer.create_state(model, tx, num_train_steps=tx_args['num_train_steps'], ckpt_dir=ckpt_dir)
  		try:
  				trainer.train(state, tr_dataset, val_dataset)
  		except KeyboardInterrupt:
  				print('Oooops; TRAINING STOPPED UNFORTUNATELY')
  
  		print('SAVING WEIGHTS IN `final-weights`')
  		lowercase_								=    jax_utils.unreplicate(state.params)
  		model.save_pretrained(os.path.join(args.base_dir, 'final-weights'), params=params)
  
  
  
style_context: 
  
  
  import numpy as np
  import PIL
  import torch
  import torchvision.transforms as T
  import torchvision.transforms.functional as TF
  from PIL import Image
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase							,     _UpperCAmelCase=256				)    ->  Dict:
  						_a     =			min(img.size				)
  
  						if s < target_image_size:
  												raise ValueError(f"""min dim for image {s} < {target_image_size}"""				)
  
  						_a     =			target_image_size / s
  						_a     =			(round(r * img.size[1]				), round(r * img.size[0]				))
  						_a     =			TF.resize(_UpperCAmelCase							,     _UpperCAmelCase							,     interpolation=PIL.Image.LANCZOS				)
  						_a     =			TF.center_crop(_UpperCAmelCase							,     output_size=2 * [target_image_size]				)
  						_a     =			torch.unsqueeze(T.ToTensor()(_UpperCAmelCase				)							,     0				)
  						return img
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase				)    ->  List[Any]:
  						_a     =			2.0 * x - 1.0
  						return x
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase							,     _UpperCAmelCase=True							,     _UpperCAmelCase="RGB"				)    ->  Optional[Any]:
  						_a     =			x.detach().cpu()
  						if process:
  												_a     =			post_process_tensor(_UpperCAmelCase				)
  						_a     =			x.numpy()
  						if process:
  												_a     =			(255 * x).astype(np.uinta				)
  						_a     =			Image.fromarray(_UpperCAmelCase				)
  						if not x.mode == mode:
  												_a     =			x.convert(_UpperCAmelCase				)
  						return x
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase				)    ->  Optional[Any]:
  						_a     =			torch.clamp(_UpperCAmelCase							,     -1.0							,     1.0				)
  						_a     =			(x + 1.0) / 2.0
  						_a     =			x.permute(1							,     2							,     0				)
  						return x
  
  
  
  
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase				)    ->  str:
  						_a     =			post_process_tensor(x.squeeze()				)
  						return x.permute(2							,     0							,     1				).unsqueeze(0				)
  
  
  