code: 
  
  
  
  
  
  
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
  						print("#################### AVAILABLE DEVICES ####################")
  						print(jax.devices())
  						print("###########################################################")
  
  						# setup for wandb sweep
  						__lowerCamelCase       :       Any				   =			Args()
  						__lowerCamelCase       :       Optional[Any]				   =			wandb.init(project="bigbird-natural-questions", config=args.__dict__)
  						__lowerCamelCase       :       Tuple				   =			dict(logger.config)
  						del wandb_args["batch_size"]
  						__lowerCamelCase       :       List[str]				   =			replace(args, **wandb_args)
  						__lowerCamelCase       :       Dict				   =			args.base_dir + "-" + wandb.run.id
  						__lowerCamelCase       :       Dict				   =			replace(args, base_dir=base_dir)
  						print(args)
  
  						__lowerCamelCase       :       List[str]				   =			load_dataset("json", data_files=args.tr_data_path)["train"]
  						__lowerCamelCase       :       Tuple				   =			load_dataset("json", data_files=args.val_data_path)["train"]
  
  						# drop extra batch for now
  						__lowerCamelCase       :       Any				   =			range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
  						__lowerCamelCase       :       Dict				   =			tr_dataset.shuffle().select(indices)
  						__lowerCamelCase       :       Optional[int]				   =			range(len(val_dataset) - len(val_dataset) % args.batch_size)
  						__lowerCamelCase       :       Union[str, Any]				   =			val_dataset.shuffle().select(indices)
  
  						if os.environ.get("TRAIN_ON_SMALL", "false") == "true":
  												__lowerCamelCase       :       Dict				   =			tr_dataset.shuffle().select(range(80000))
  												__lowerCamelCase       :       List[Any]				   =			val_dataset.shuffle().select(range(8000))
  
  						print(tr_dataset)
  						print(val_dataset)
  
  						__lowerCamelCase       :       int				   =			FlaxBigBirdForNaturalQuestions.from_pretrained(
  						    args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
  						)
  						__lowerCamelCase       :       Any				   =			BigBirdTokenizerFast.from_pretrained(args.model_id)
  						__lowerCamelCase       :       List[str]				   =			DataCollator(pad_id=tokenizer.pad_token_id, max_length=4096)
  
  						__lowerCamelCase       :       Union[str, Any]				   =			{
  						    "lr": args.lr,
  						    "init_lr": args.init_lr,
  						    "warmup_steps": args.warmup_steps,
  						    "num_train_steps": args.max_epochs * (len(tr_dataset) // args.batch_size),
  						    "weight_decay": args.weight_decay,
  						}
  						__lowerCamelCase   ,   __lowerCamelCase       :       List[str]				   =			build_tx(**tx_args)
  
  						__lowerCamelCase       :       Tuple				   =			Trainer(
  						    args=args,
  						    data_collator=data_collator,
  						    model_save_fn=model.save_pretrained,
  						    train_step_fn=train_step,
  						    val_step_fn=val_step,
  						    logger=logger,
  						    scheduler_fn=lr,
  						)
  
  						__lowerCamelCase       :       str				   =			None
  						__lowerCamelCase       :       Optional[int]				   =			trainer.create_state(model, tx, num_train_steps=tx_args["num_train_steps"], ckpt_dir=ckpt_dir)
  						try:
  												trainer.train(state, tr_dataset, val_dataset)
  						except KeyboardInterrupt:
  												print("Oooops; TRAINING STOPPED UNFORTUNATELY")
  
  						print("SAVING WEIGHTS IN `final-weights`")
  						__lowerCamelCase       :       List[Any]				   =			jax_utils.unreplicate(state.params)
  						model.save_pretrained(os.path.join(args.base_dir, "final-weights"), params=params)
  
  
  
  
style_context: 
  
  
  import argparse
  import collections
  from pathlib import Path
  
  import torch
  from torch.serialization import default_restore_location
  
  from transformers import BertConfig, DPRConfig, DPRContextEncoder, DPRQuestionEncoder, DPRReader
  
  
  __lowerCamelCase       :       Optional[Any]				   =			collections.namedtuple(
      "CheckpointState", ["model_dict", "optimizer_dict", "scheduler_dict", "offset", "epoch", "encoder_params"]
  )
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_	)			->							CheckpointState:
  				print(f'Reading saved model from {model_file}'	)
  				A__     :							Optional[int]   				=torch.load(snake_case_,	map_location=lambda snake_case_,	snake_case_	: default_restore_location(snake_case_,	'''cpu'''	)	)
  				return CheckpointState(**snake_case_	)
  
  
  
  class 			a      :
  
  
  
  
  
  
  		def __init__(  self	,     __UpperCamelCase							)->			Union[str, Any]:
  						'''simple docstring'''
  
  
  						A__     :							Dict   				=src_file
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Optional[Any]:
  						'''simple docstring'''
  
  
  						raise NotImplementedError
  
  
  
  
  
  
  		@staticmethod
  		def       lowerCAmelCase_  (  __UpperCamelCase	,     *__UpperCamelCase	,     **__UpperCamelCase							)->			"DPRState":
  						'''simple docstring'''
  
  
  						if comp_type.startswith('''c'''							):
  										return DPRContextEncoderState(*__UpperCamelCase	,     **__UpperCamelCase							)
  						if comp_type.startswith('''q'''							):
  										return DPRQuestionEncoderState(*__UpperCamelCase	,     **__UpperCamelCase							)
  						if comp_type.startswith('''r'''							):
  										return DPRReaderState(*__UpperCamelCase	,     **__UpperCamelCase							)
  						else:
  										raise ValueError('''Component type must be either \'ctx_encoder\', \'question_encoder\' or \'reader\'.'''							)
  
  
  
  
  
  
  
  class 			a      (      UpperCamelCase_   ):
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Optional[Any]:
  						'''simple docstring'''
  
  
  						A__     :							Optional[int]   				=DPRContextEncoder(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased'''							)[0]							)							)
  						print(F'Loading DPR biencoder from {self.src_file}'							)
  						A__     :							Optional[int]   				=load_states_from_checkpoint(self.src_file							)
  						A__				,  A__     :							List[str]   				=model.ctx_encoder, '''ctx_model.'''
  						# Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
  						A__     :							Optional[int]   				={'''bert_model.embeddings.position_ids''': model.ctx_encoder.bert_model.embeddings.position_ids}
  						for key, value in saved_state.model_dict.items():
  										if key.startswith(__UpperCamelCase							):
  														A__     :							int   				=key[len(__UpperCamelCase							) :]
  														if not key.startswith('''encode_proj.'''							):
  																		A__     :							Any   				='''bert_model.''' + key
  														A__     :							List[str]   				=value
  						encoder.load_state_dict(__UpperCamelCase							)
  						return model
  
  
  
  
  
  
  
  class 			a      (      UpperCamelCase_   ):
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Dict:
  						'''simple docstring'''
  
  
  						A__     :							Union[str, Any]   				=DPRQuestionEncoder(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased'''							)[0]							)							)
  						print(F'Loading DPR biencoder from {self.src_file}'							)
  						A__     :							List[str]   				=load_states_from_checkpoint(self.src_file							)
  						A__				,  A__     :							Optional[int]   				=model.question_encoder, '''question_model.'''
  						# Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
  						A__     :							Optional[Any]   				={'''bert_model.embeddings.position_ids''': model.question_encoder.bert_model.embeddings.position_ids}
  						for key, value in saved_state.model_dict.items():
  										if key.startswith(__UpperCamelCase							):
  														A__     :							List[Any]   				=key[len(__UpperCamelCase							) :]
  														if not key.startswith('''encode_proj.'''							):
  																		A__     :							Tuple   				='''bert_model.''' + key
  														A__     :							str   				=value
  						encoder.load_state_dict(__UpperCamelCase							)
  						return model
  
  
  
  
  
  
  
  class 			a      (      UpperCamelCase_   ):
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			int:
  						'''simple docstring'''
  
  
  						A__     :							Optional[int]   				=DPRReader(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased'''							)[0]							)							)
  						print(F'Loading DPR reader from {self.src_file}'							)
  						A__     :							Dict   				=load_states_from_checkpoint(self.src_file							)
  						# Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
  						A__     :							Optional[Any]   				={
  						    '''encoder.bert_model.embeddings.position_ids''': model.span_predictor.encoder.bert_model.embeddings.position_ids
  						}
  						for key, value in saved_state.model_dict.items():
  										if key.startswith('''encoder.'''							) and not key.startswith('''encoder.encode_proj'''							):
  														A__     :							List[Any]   				='''encoder.bert_model.''' + key[len('''encoder.'''							) :]
  										A__     :							Optional[int]   				=value
  						model.span_predictor.load_state_dict(__UpperCamelCase							)
  						return model
  
  
  
  
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_,	snake_case_,	snake_case_	)			->							Optional[int]:
  				A__     :							Optional[int]   				=Path(snake_case_	)
  				dest_dir.mkdir(exist_ok=snake_case_	)
  
  				A__     :							Tuple   				=DPRState.from_type(snake_case_,	src_file=snake_case_	)
  				A__     :							str   				=dpr_state.load_dpr_model()
  				model.save_pretrained(snake_case_	)
  				model.from_pretrained(snake_case_	)  # sanity check
  
  
  if __name__ == "__main__":
  						__lowerCamelCase       :       Optional[Any]				   =			argparse.ArgumentParser()
  						# Required parameters
  						parser.add_argument(
  						    "--type", type=str, help="Type of the component to convert: 'ctx_encoder', 'question_encoder' or 'reader'."
  						)
  						parser.add_argument(
  						    "--src",
  						    type=str,
  						    help=(
  						        "Path to the dpr checkpoint file. They can be downloaded from the official DPR repo"
  						        " https://github.com/facebookresearch/DPR. Note that in the official repo, both encoders are stored in the"
  						        " 'retriever' checkpoints."
  						    ),
  						)
  						parser.add_argument("--dest", type=str, default=None, help="Path to the output PyTorch model directory.")
  						__lowerCamelCase       :       Tuple				   =			parser.parse_args()
  
  						__lowerCamelCase       :       Optional[Any]				   =			Path(args.src)
  						__lowerCamelCase       :       Optional[int]				   =			F"converted-{src_file.name}" if args.dest is None else args.dest
  						__lowerCamelCase       :       Optional[Any]				   =			Path(dest_dir)
  						assert src_file.exists()
  						assert (
  						    args.type is not None
  						), "Please specify the component type of the DPR model to convert: 'ctx_encoder', 'question_encoder' or 'reader'."
  						convert(args.type, src_file, dest_dir)
  
  
  
  