code: 
  
  
  
  
  
  
  import enum
  import inspect
  from typing import Iterable, List, Optional, Tuple, Union
  
  
  
  
  
  class     __a						(					enum.Enum ):
   _lowerCAmelCase : List[Any] 				=				'''timm'''
   _lowerCAmelCase : Any 				=				'''transformers'''
  
  
  
  
  
  
  
  def 	SCREAMING_SNAKE_CASE							(      __lowerCAmelCase      ,					__lowerCAmelCase      ,					__lowerCAmelCase    )				->			Optional[Any]:
       if stage_names is None:
            raise ValueError("Stage_names must be set for transformers backbones"    )
  
       if out_features is not None:
            if not isinstance(__lowerCAmelCase      ,					(list,)    ):
                 raise ValueError(f'out_features must be a list {type(__lowerCAmelCase    )}'    )
            if any(feat not in stage_names for feat in out_features    ):
                 raise ValueError(f'out_features must be a subset of stage_names: {stage_names} got {out_features}'    )
  
       if out_indices is not None:
            if not isinstance(__lowerCAmelCase      ,					(list, tuple)    ):
                 raise ValueError(f'out_indices must be a list or tuple, got {type(__lowerCAmelCase    )}'    )
            if any(idx >= len(__lowerCAmelCase    ) for idx in out_indices    ):
                 raise ValueError("out_indices must be valid indices for stage_names {stage_names}, got {out_indices}"    )
  
       if out_features is not None and out_indices is not None:
            if len(__lowerCAmelCase    ) != len(__lowerCAmelCase    ):
                 raise ValueError("out_features and out_indices should have the same length if both are set"    )
            if out_features != [stage_names[idx] for idx in out_indices]:
                 raise ValueError("out_features and out_indices should correspond to the same stages if both are set"    )
  
  
  
  
  
  
  
  def 	SCREAMING_SNAKE_CASE							(      __lowerCAmelCase      ,					__lowerCAmelCase      ,					__lowerCAmelCase      ,					)				->			List[str]:
       if out_indices is None and out_features is None:
            UpperCamelCase__    :     Dict        =						[len(__lowerCAmelCase    ) - 1]
            UpperCamelCase__    :     Optional[int]        =						[stage_names[-1]]
       elif out_indices is None and out_features is not None:
            UpperCamelCase__    :     List[Any]        =						[stage_names.index(__lowerCAmelCase    ) for layer in out_features]
       elif out_features is None and out_indices is not None:
            UpperCamelCase__    :     Any        =						[stage_names[idx] for idx in out_indices]
       return out_features, out_indices
  
  
  
  
  
  
  
  def 	SCREAMING_SNAKE_CASE							(      __lowerCAmelCase      ,					__lowerCAmelCase      ,					__lowerCAmelCase      ,					)				->			Tuple[List[str], List[int]]:
       verify_out_features_out_indices(out_features=__lowerCAmelCase      ,					out_indices=__lowerCAmelCase      ,					stage_names=__lowerCAmelCase    )
       UpperCamelCase__       ,  UpperCamelCase__    :     Tuple        =						_align_output_features_output_indices(
           out_features=__lowerCAmelCase      ,					out_indices=__lowerCAmelCase      ,					stage_names=__lowerCAmelCase    )
       # Verify that the aligned out_features and out_indices are valid
       verify_out_features_out_indices(out_features=__lowerCAmelCase      ,					out_indices=__lowerCAmelCase      ,					stage_names=__lowerCAmelCase    )
       return output_features, output_indices
  
  
  
  
  
  class     __a						:
   _lowerCAmelCase : Optional[BackboneType] 				=				None
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     str							,     SCREAMING_SNAKE_CASE      :     Dict		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        if getattr(self							,     "_backbone"							,     SCREAMING_SNAKE_CASE		) is None:
             raise ValueError("self._backbone must be set before calling _init_timm_backbone"		)
  
        # These will diagree with the defaults for the transformers models e.g. for resnet50
        # the transformer model has out_features = ['stem', 'stage1', 'stage2', 'stage3', 'stage4']
        # the timm model has out_features = ['act', 'layer1', 'layer2', 'layer3', 'layer4']
        UpperCamelCase__    :     Any        =						[stage["module"] for stage in self._backbone.feature_info.info]
        UpperCamelCase__    :     List[str]        =						[stage["num_chs"] for stage in self._backbone.feature_info.info]
        UpperCamelCase__    :     Union[str, Any]        =						self._backbone.feature_info.out_indices
        UpperCamelCase__    :     Union[str, Any]        =						self._backbone.feature_info.module_name()
  
        # We verify the out indices and out features are valid
        verify_out_features_out_indices(
            out_features=SCREAMING_SNAKE_CASE							,     out_indices=SCREAMING_SNAKE_CASE							,     stage_names=self.stage_names		)
        UpperCamelCase__       ,  UpperCamelCase__    :     Union[str, Any]        =						out_features, out_indices
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     int							,     SCREAMING_SNAKE_CASE      :     Union[str, Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     List[str]        =						getattr(SCREAMING_SNAKE_CASE							,     "stage_names"		)
        UpperCamelCase__    :     List[str]        =						getattr(SCREAMING_SNAKE_CASE							,     "out_features"							,     SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     List[Any]        =						getattr(SCREAMING_SNAKE_CASE							,     "out_indices"							,     SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     Any        =						stage_names
        UpperCamelCase__       ,  UpperCamelCase__    :     str        =						get_aligned_output_features_output_indices(
            out_features=SCREAMING_SNAKE_CASE							,     out_indices=SCREAMING_SNAKE_CASE							,     stage_names=SCREAMING_SNAKE_CASE		)
        # Number of channels for each stage. This is set in the transformer backbone model init
        UpperCamelCase__    :     Tuple        =						None
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[int]							,     SCREAMING_SNAKE_CASE      :     List[str]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Any        =						config
  
        UpperCamelCase__    :     str        =						getattr(SCREAMING_SNAKE_CASE							,     "use_timm_backbone"							,     SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Optional[int]        =						BackboneType.TIMM if self.use_timm_backbone else BackboneType.TRANSFORMERS
  
        if self.backbone_type == BackboneType.TIMM:
             self._init_timm_backbone(SCREAMING_SNAKE_CASE		)
        elif self.backbone_type == BackboneType.TRANSFORMERS:
             self._init_transformers_backbone(SCREAMING_SNAKE_CASE		)
        else:
             raise ValueError(F'backbone_type {self.backbone_type} not supported.'		)
  
  
  
  
  
  
  
   @property
   def       __lowercase      (	self      :     List[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        return self._out_features
  
  
  
  
  
  
  
   @out_features.setter
   def       __lowercase      (	self      :     Dict							,     SCREAMING_SNAKE_CASE      :     List[str]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__       ,  UpperCamelCase__    :     str        =						get_aligned_output_features_output_indices(
            out_features=SCREAMING_SNAKE_CASE							,     out_indices=SCREAMING_SNAKE_CASE							,     stage_names=self.stage_names		)
  
  
  
  
  
  
  
   @property
   def       __lowercase      (	self      :     Optional[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        return self._out_indices
  
  
  
  
  
  
  
   @out_indices.setter
   def       __lowercase      (	self      :     Optional[int]							,     SCREAMING_SNAKE_CASE      :     Union[Tuple[int], List[int]]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__       ,  UpperCamelCase__    :     str        =						get_aligned_output_features_output_indices(
            out_features=SCREAMING_SNAKE_CASE							,     out_indices=SCREAMING_SNAKE_CASE							,     stage_names=self.stage_names		)
  
  
  
  
  
  
  
   @property
   def       __lowercase      (	self      :     int		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        return {stage: self.num_features[i] for i, stage in enumerate(self.stage_names		)}
  
  
  
  
  
  
  
   @property
   def       __lowercase      (	self      :     Optional[int]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        return [self.out_feature_channels[name] for name in self.out_features]
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     List[str]							,     *SCREAMING_SNAKE_CASE      :     List[str]							,     **SCREAMING_SNAKE_CASE      :     str		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     List[str]        =						dict(inspect.signature(self.forward		).parameters		)
        UpperCamelCase__    :     Optional[int]        =						{k: v for k, v in kwargs.items() if k in signature}
        return self(*SCREAMING_SNAKE_CASE							,     **SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     List[str]							,     SCREAMING_SNAKE_CASE      :     List[str]							,     SCREAMING_SNAKE_CASE      :     Optional[bool] = None							,     SCREAMING_SNAKE_CASE      :     Optional[bool] = None							,     SCREAMING_SNAKE_CASE      :     Optional[bool] = None							,     ):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        raise NotImplementedError("This method should be implemented by the derived class."		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Tuple		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Tuple        =						super().to_dict()
        UpperCamelCase__    :     Optional[int]        =						output.pop("_out_features"		)
        UpperCamelCase__    :     int        =						output.pop("_out_indices"		)
        return output
  
  
  
  
  
  
  
  class     __a						:
  
  
  
  
  
  
  
   @property
   def       __lowercase      (	self      :     Union[str, Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        return self._out_features
  
  
  
  
  
  
  
   @out_features.setter
   def       __lowercase      (	self      :     Any							,     SCREAMING_SNAKE_CASE      :     List[str]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__       ,  UpperCamelCase__    :     str        =						get_aligned_output_features_output_indices(
            out_features=SCREAMING_SNAKE_CASE							,     out_indices=SCREAMING_SNAKE_CASE							,     stage_names=self.stage_names		)
  
  
  
  
  
  
  
   @property
   def       __lowercase      (	self      :     Optional[int]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        return self._out_indices
  
  
  
  
  
  
  
   @out_indices.setter
   def       __lowercase      (	self      :     List[str]							,     SCREAMING_SNAKE_CASE      :     Union[Tuple[int], List[int]]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__       ,  UpperCamelCase__    :     Dict        =						get_aligned_output_features_output_indices(
            out_features=SCREAMING_SNAKE_CASE							,     out_indices=SCREAMING_SNAKE_CASE							,     stage_names=self.stage_names		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Union[str, Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Dict        =						super().to_dict()
        UpperCamelCase__    :     str        =						output.pop("_out_features"		)
        UpperCamelCase__    :     List[Any]        =						output.pop("_out_indices"		)
        return output
style_context: 
  
  
  
  
  
  
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
   print('''#################### AVAILABLE DEVICES ####################''')
   print(jax.devices())
   print('''###########################################################''')
  
   # setup for wandb sweep
   lowerCamelCase			:							str       =Args()
   lowerCamelCase			:							str       =wandb.init(project='''bigbird-natural-questions''', config=args.__dict__)
   lowerCamelCase			:							Any       =dict(logger.config)
   del wandb_args["batch_size"]
   lowerCamelCase			:							str       =replace(args, **wandb_args)
   lowerCamelCase			:							List[str]       =args.base_dir + '''-''' + wandb.run.id
   lowerCamelCase			:							str       =replace(args, base_dir=base_dir)
   print(args)
  
   lowerCamelCase			:							List[Any]       =load_dataset('''json''', data_files=args.tr_data_path)['''train''']
   lowerCamelCase			:							str       =load_dataset('''json''', data_files=args.val_data_path)['''train''']
  
   # drop extra batch for now
   lowerCamelCase			:							List[Any]       =range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
   lowerCamelCase			:							Union[str, Any]       =tr_dataset.shuffle().select(indices)
   lowerCamelCase			:							List[str]       =range(len(val_dataset) - len(val_dataset) % args.batch_size)
   lowerCamelCase			:							Dict       =val_dataset.shuffle().select(indices)
  
   if os.environ.get('''TRAIN_ON_SMALL''', '''false''') == "true":
    lowerCamelCase			:							Optional[int]       =tr_dataset.shuffle().select(range(80000))
    lowerCamelCase			:							List[Any]       =val_dataset.shuffle().select(range(8000))
  
   print(tr_dataset)
   print(val_dataset)
  
   lowerCamelCase			:							int       =FlaxBigBirdForNaturalQuestions.from_pretrained(
       args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
   )
   lowerCamelCase			:							Union[str, Any]       =BigBirdTokenizerFast.from_pretrained(args.model_id)
   lowerCamelCase			:							Dict       =DataCollator(pad_id=tokenizer.pad_token_id, max_length=4096)
  
   lowerCamelCase			:							List[str]       ={
       '''lr''': args.lr,
       '''init_lr''': args.init_lr,
       '''warmup_steps''': args.warmup_steps,
       '''num_train_steps''': args.max_epochs * (len(tr_dataset) // args.batch_size),
       '''weight_decay''': args.weight_decay,
   }
   lowerCamelCase       ,    lowerCamelCase			:							Tuple       =build_tx(**tx_args)
  
   lowerCamelCase			:							Optional[Any]       =Trainer(
       args=args,
       data_collator=data_collator,
       model_save_fn=model.save_pretrained,
       train_step_fn=train_step,
       val_step_fn=val_step,
       logger=logger,
       scheduler_fn=lr,
   )
  
   lowerCamelCase			:							Tuple       =None
   lowerCamelCase			:							str       =trainer.create_state(model, tx, num_train_steps=tx_args['''num_train_steps'''], ckpt_dir=ckpt_dir)
   try:
    trainer.train(state, tr_dataset, val_dataset)
   except KeyboardInterrupt:
    print('''Oooops; TRAINING STOPPED UNFORTUNATELY''')
  
   print('''SAVING WEIGHTS IN `final-weights`''')
   lowerCamelCase			:							str       =jax_utils.unreplicate(state.params)
   model.save_pretrained(os.path.join(args.base_dir, '''final-weights'''), params=params)