code: 
  
  
  
  
  
  
  from typing import Dict, List, Optional, Union
  
  import numpy as np
  
  from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
  from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
  from ...image_utils import (
      IMAGENET_DEFAULT_MEAN,
      IMAGENET_DEFAULT_STD,
      ChannelDimension,
      ImageInput,
      PILImageResampling,
      get_image_size,
      make_list_of_images,
      to_numpy_array,
      valid_images,
  )
  from ...utils import TensorType, is_vision_available, logging
  
  
  if is_vision_available():
   import PIL
  
  
  lowerCamelCase			:							Union[str, Any]       =logging.get_logger(__name__)
  
  
  
  
  
  class     __a						(					A__ ):
   _lowerCAmelCase : List[Any] 				=				['''pixel_values''']
  
  
  
  
  
  
  
   def __init__(	self      :     List[str]							,     SCREAMING_SNAKE_CASE      :     bool = True							,     SCREAMING_SNAKE_CASE      :     Dict[str, int] = None							,     SCREAMING_SNAKE_CASE      :     bool = True							,     SCREAMING_SNAKE_CASE      :     Dict[str, int] = None							,     SCREAMING_SNAKE_CASE      :     PILImageResampling = PILImageResampling.BICUBIC							,     SCREAMING_SNAKE_CASE      :     bool = True							,     SCREAMING_SNAKE_CASE      :     Union[int, float] = 1 / 2_55							,     SCREAMING_SNAKE_CASE      :     bool = True							,     SCREAMING_SNAKE_CASE      :     Optional[Union[float, List[float]]] = None							,     SCREAMING_SNAKE_CASE      :     Optional[Union[float, List[float]]] = None							,     **SCREAMING_SNAKE_CASE      :     Optional[int]							,     ):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        super().__init__(**SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Optional[Any]        =						crop_size if crop_size is not None else {"height": 2_56, "width": 2_56}
        UpperCamelCase__    :     int        =						get_size_dict(SCREAMING_SNAKE_CASE							,     param_name="crop_size"		)
        UpperCamelCase__    :     Optional[int]        =						size if size is not None else {"height": 2_24, "width": 2_24}
        UpperCamelCase__    :     Union[str, Any]        =						get_size_dict(SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     Optional[Any]        =						do_center_crop
        UpperCamelCase__    :     Any        =						crop_size
        UpperCamelCase__    :     Union[str, Any]        =						do_resize
        UpperCamelCase__    :     Optional[int]        =						size
        UpperCamelCase__    :     Optional[int]        =						resample
        UpperCamelCase__    :     Union[str, Any]        =						do_rescale
        UpperCamelCase__    :     Dict        =						rescale_factor
        UpperCamelCase__    :     Optional[Any]        =						do_normalize
        UpperCamelCase__    :     List[Any]        =						image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
        UpperCamelCase__    :     List[Any]        =						image_std if image_std is not None else IMAGENET_DEFAULT_STD
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     str							,     SCREAMING_SNAKE_CASE      :     np.ndarray							,     SCREAMING_SNAKE_CASE      :     Dict[str, int]							,     SCREAMING_SNAKE_CASE      :     Optional[int] = None							,     SCREAMING_SNAKE_CASE      :     Optional[Union[str, ChannelDimension]] = None							,     **SCREAMING_SNAKE_CASE      :     Optional[Any]							,     ):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     List[Any]        =						self.size if size is None else size
        UpperCamelCase__    :     Optional[int]        =						get_size_dict(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     List[str]        =						get_size_dict(SCREAMING_SNAKE_CASE							,     param_name="crop_size"		)
  
        UpperCamelCase__       ,  UpperCamelCase__    :     int        =						get_image_size(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Optional[Any]        =						min(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     List[Any]        =						(size["height"] / crop_size["height"]) * min_dim
        UpperCamelCase__    :     List[str]        =						(size["width"] / crop_size["width"]) * min_dim
        return center_crop(SCREAMING_SNAKE_CASE							,     size=(cropped_height, cropped_width)							,     data_format=SCREAMING_SNAKE_CASE							,     **SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Union[str, Any]							,     SCREAMING_SNAKE_CASE      :     np.ndarray							,     SCREAMING_SNAKE_CASE      :     Dict[str, int]							,     SCREAMING_SNAKE_CASE      :     PILImageResampling = PIL.Image.BICUBIC							,     SCREAMING_SNAKE_CASE      :     Optional[Union[str, ChannelDimension]] = None							,     **SCREAMING_SNAKE_CASE      :     Any							,     ):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     int        =						get_size_dict(SCREAMING_SNAKE_CASE		)
        if "height" not in size or "width" not in size:
             raise ValueError(F'The size dictionary must contain the keys \'height\' and \'width\'. Got {size.keys()}'		)
        return resize(
            SCREAMING_SNAKE_CASE							,     size=(size["height"], size["width"])							,     resample=SCREAMING_SNAKE_CASE							,     data_format=SCREAMING_SNAKE_CASE							,     **SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Any							,     SCREAMING_SNAKE_CASE      :     np.ndarray							,     SCREAMING_SNAKE_CASE      :     Union[int, float]							,     SCREAMING_SNAKE_CASE      :     Optional[Union[str, ChannelDimension]] = None							,     **SCREAMING_SNAKE_CASE      :     Optional[Any]							,     ):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        return rescale(SCREAMING_SNAKE_CASE							,     scale=SCREAMING_SNAKE_CASE							,     data_format=SCREAMING_SNAKE_CASE							,     **SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Dict							,     SCREAMING_SNAKE_CASE      :     np.ndarray							,     SCREAMING_SNAKE_CASE      :     Union[float, List[float]]							,     SCREAMING_SNAKE_CASE      :     Union[float, List[float]]							,     SCREAMING_SNAKE_CASE      :     Optional[Union[str, ChannelDimension]] = None							,     **SCREAMING_SNAKE_CASE      :     Optional[int]							,     ):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        return normalize(SCREAMING_SNAKE_CASE							,     mean=SCREAMING_SNAKE_CASE							,     std=SCREAMING_SNAKE_CASE							,     data_format=SCREAMING_SNAKE_CASE							,     **SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Union[str, Any]							,     SCREAMING_SNAKE_CASE      :     ImageInput							,     SCREAMING_SNAKE_CASE      :     Optional[bool] = None							,     SCREAMING_SNAKE_CASE      :     Optional[Dict[str, int]] = None							,     SCREAMING_SNAKE_CASE      :     Optional[bool] = None							,     SCREAMING_SNAKE_CASE      :     Optional[Dict[str, int]] = None							,     SCREAMING_SNAKE_CASE      :     PILImageResampling = None							,     SCREAMING_SNAKE_CASE      :     Optional[bool] = None							,     SCREAMING_SNAKE_CASE      :     Optional[float] = None							,     SCREAMING_SNAKE_CASE      :     Optional[bool] = None							,     SCREAMING_SNAKE_CASE      :     Optional[Union[float, List[float]]] = None							,     SCREAMING_SNAKE_CASE      :     Optional[Union[float, List[float]]] = None							,     SCREAMING_SNAKE_CASE      :     Optional[Union[str, TensorType]] = None							,     SCREAMING_SNAKE_CASE      :     ChannelDimension = ChannelDimension.FIRST							,     **SCREAMING_SNAKE_CASE      :     str							,     ):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     List[str]        =						do_center_crop if do_center_crop is not None else self.do_center_crop
        UpperCamelCase__    :     Tuple        =						crop_size if crop_size is not None else self.crop_size
        UpperCamelCase__    :     Optional[Any]        =						get_size_dict(SCREAMING_SNAKE_CASE							,     param_name="crop_size"		)
        UpperCamelCase__    :     List[str]        =						do_resize if do_resize is not None else self.do_resize
        UpperCamelCase__    :     List[Any]        =						size if size is not None else self.size
        UpperCamelCase__    :     Tuple        =						get_size_dict(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Union[str, Any]        =						resample if resample is not None else self.resample
        UpperCamelCase__    :     int        =						do_rescale if do_rescale is not None else self.do_rescale
        UpperCamelCase__    :     Optional[int]        =						rescale_factor if rescale_factor is not None else self.rescale_factor
        UpperCamelCase__    :     Union[str, Any]        =						do_normalize if do_normalize is not None else self.do_normalize
        UpperCamelCase__    :     str        =						image_mean if image_mean is not None else self.image_mean
        UpperCamelCase__    :     List[Any]        =						image_std if image_std is not None else self.image_std
  
        UpperCamelCase__    :     Union[str, Any]        =						make_list_of_images(SCREAMING_SNAKE_CASE		)
  
        if not valid_images(SCREAMING_SNAKE_CASE		):
             raise ValueError(
                 "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
                 "torch.Tensor, tf.Tensor or jax.ndarray."		)
  
        if do_center_crop and crop_size is None:
             raise ValueError("If `do_center_crop` is set to `True`, `crop_size` must be provided."		)
  
        if do_resize and size is None:
             raise ValueError("Size must be specified if do_resize is True."		)
  
        if do_rescale and rescale_factor is None:
             raise ValueError("Rescale factor must be specified if do_rescale is True."		)
  
        if do_normalize and (image_mean is None or image_std is None):
             raise ValueError("Image mean and image standard deviation must be specified if do_normalize is True."		)
  
        # All transformations expect numpy arrays.
        UpperCamelCase__    :     Dict        =						[to_numpy_array(SCREAMING_SNAKE_CASE		) for image in images]
  
        if do_center_crop:
             UpperCamelCase__    :     Tuple        =						[self.center_crop(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     size=SCREAMING_SNAKE_CASE		) for image in images]
  
        if do_resize:
             UpperCamelCase__    :     Optional[int]        =						[self.resize(image=SCREAMING_SNAKE_CASE							,     size=SCREAMING_SNAKE_CASE							,     resample=SCREAMING_SNAKE_CASE		) for image in images]
  
        if do_rescale:
             UpperCamelCase__    :     Any        =						[self.rescale(image=SCREAMING_SNAKE_CASE							,     scale=SCREAMING_SNAKE_CASE		) for image in images]
  
        if do_normalize:
             UpperCamelCase__    :     List[str]        =						[self.normalize(image=SCREAMING_SNAKE_CASE							,     mean=SCREAMING_SNAKE_CASE							,     std=SCREAMING_SNAKE_CASE		) for image in images]
  
        UpperCamelCase__    :     Tuple        =						[to_channel_dimension_format(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		) for image in images]
  
        UpperCamelCase__    :     Optional[Any]        =						{"pixel_values": images}
        return BatchFeature(data=SCREAMING_SNAKE_CASE							,     tensor_type=SCREAMING_SNAKE_CASE		)
style_context: 
  
  
  
  
  
  
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
   print('''#################### AVAILABLE DEVICES ####################''')
   print(jax.devices())
   print('''###########################################################''')
  
   # setup for wandb sweep
   lowerCamelCase			:							str       =Args()
   lowerCamelCase			:							str       =wandb.init(project='''bigbird-natural-questions''', config=args.__dict__)
   lowerCamelCase			:							Any       =dict(logger.config)
   del wandb_args["batch_size"]
   lowerCamelCase			:							str       =replace(args, **wandb_args)
   lowerCamelCase			:							List[str]       =args.base_dir + '''-''' + wandb.run.id
   lowerCamelCase			:							str       =replace(args, base_dir=base_dir)
   print(args)
  
   lowerCamelCase			:							List[Any]       =load_dataset('''json''', data_files=args.tr_data_path)['''train''']
   lowerCamelCase			:							str       =load_dataset('''json''', data_files=args.val_data_path)['''train''']
  
   # drop extra batch for now
   lowerCamelCase			:							List[Any]       =range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
   lowerCamelCase			:							Union[str, Any]       =tr_dataset.shuffle().select(indices)
   lowerCamelCase			:							List[str]       =range(len(val_dataset) - len(val_dataset) % args.batch_size)
   lowerCamelCase			:							Dict       =val_dataset.shuffle().select(indices)
  
   if os.environ.get('''TRAIN_ON_SMALL''', '''false''') == "true":
    lowerCamelCase			:							Optional[int]       =tr_dataset.shuffle().select(range(80000))
    lowerCamelCase			:							List[Any]       =val_dataset.shuffle().select(range(8000))
  
   print(tr_dataset)
   print(val_dataset)
  
   lowerCamelCase			:							int       =FlaxBigBirdForNaturalQuestions.from_pretrained(
       args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
   )
   lowerCamelCase			:							Union[str, Any]       =BigBirdTokenizerFast.from_pretrained(args.model_id)
   lowerCamelCase			:							Dict       =DataCollator(pad_id=tokenizer.pad_token_id, max_length=4096)
  
   lowerCamelCase			:							List[str]       ={
       '''lr''': args.lr,
       '''init_lr''': args.init_lr,
       '''warmup_steps''': args.warmup_steps,
       '''num_train_steps''': args.max_epochs * (len(tr_dataset) // args.batch_size),
       '''weight_decay''': args.weight_decay,
   }
   lowerCamelCase       ,    lowerCamelCase			:							Tuple       =build_tx(**tx_args)
  
   lowerCamelCase			:							Optional[Any]       =Trainer(
       args=args,
       data_collator=data_collator,
       model_save_fn=model.save_pretrained,
       train_step_fn=train_step,
       val_step_fn=val_step,
       logger=logger,
       scheduler_fn=lr,
   )
  
   lowerCamelCase			:							Tuple       =None
   lowerCamelCase			:							str       =trainer.create_state(model, tx, num_train_steps=tx_args['''num_train_steps'''], ckpt_dir=ckpt_dir)
   try:
    trainer.train(state, tr_dataset, val_dataset)
   except KeyboardInterrupt:
    print('''Oooops; TRAINING STOPPED UNFORTUNATELY''')
  
   print('''SAVING WEIGHTS IN `final-weights`''')
   lowerCamelCase			:							str       =jax_utils.unreplicate(state.params)
   model.save_pretrained(os.path.join(args.base_dir, '''final-weights'''), params=params)