code: 
  
  
  
  
  
  
  import os
  from typing import Optional, Union
  
  import numpy as np
  from huggingface_hub import hf_hub_download
  
  from ... import AutoTokenizer
  from ...utils import logging
  
  
  lowerCamelCase			:							List[Any]       =    '''block_records.npy'''
  
  
  lowerCamelCase			:							Tuple       =    logging.get_logger(__name__)
  
  
  
  
  
  
  
  def 	SCREAMING_SNAKE_CASE							(      __lowerCAmelCase      ,					__lowerCAmelCase    )				->			np.ndarray:
       import tensorflow.compat.va as tf
  
       UpperCamelCase__    :     Union[str, Any]        =						tf.data.TFRecordDataset(__lowerCAmelCase      ,					buffer_size=512 * 1024 * 1024    )
       UpperCamelCase__    :     List[str]        =						blocks_dataset.batch(__lowerCAmelCase      ,					drop_remainder=__lowerCAmelCase    )
       UpperCamelCase__    :     Any        =						next(blocks_dataset.take(1    ).as_numpy_iterator()    )
  
       return np_record
  
  
  
  
  
  class     __a						:
  
  
  
  
  
  
  
   def __init__(	self      :     str							,     SCREAMING_SNAKE_CASE      :     str							,     SCREAMING_SNAKE_CASE      :     Any							,     SCREAMING_SNAKE_CASE      :     Optional[int]=2							,     SCREAMING_SNAKE_CASE      :     Optional[Any]=10_00							,     SCREAMING_SNAKE_CASE      :     Optional[Any]=1_00							,     SCREAMING_SNAKE_CASE      :     int=10_00_00							,     ):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        from scann.scann_ops.py.scann_ops_pybind import builder as Builder
  
        UpperCamelCase__    :     Any        =						Builder(db=SCREAMING_SNAKE_CASE							,     num_neighbors=SCREAMING_SNAKE_CASE							,     distance_measure="dot_product"		)
        UpperCamelCase__    :     List[str]        =						builder.tree(
            num_leaves=SCREAMING_SNAKE_CASE							,     num_leaves_to_search=SCREAMING_SNAKE_CASE							,     training_sample_size=SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Tuple        =						builder.score_ah(dimensions_per_block=SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     Union[str, Any]        =						builder.build()
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     List[str]							,     SCREAMING_SNAKE_CASE      :     Union[str, Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Tuple        =						self.searcher.search_batched(question_projection.detach().cpu()		)
        return retrieved_block_ids.astype("int64"		)
  
  
  
  
  
  
  
  class     __a						:
  
  
  
  
  
  
  
   def __init__(	self      :     Any							,     SCREAMING_SNAKE_CASE      :     Optional[int]							,     SCREAMING_SNAKE_CASE      :     List[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        super().__init__()
        UpperCamelCase__    :     str        =						block_records
        UpperCamelCase__    :     List[str]        =						tokenizer
  
  
  
  
  
  
  
   def __call__(	self      :     Optional[Any]							,     SCREAMING_SNAKE_CASE      :     List[Any]							,     SCREAMING_SNAKE_CASE      :     Any							,     SCREAMING_SNAKE_CASE      :     Optional[int]							,     SCREAMING_SNAKE_CASE      :     int=None							,     SCREAMING_SNAKE_CASE      :     Dict="pt"		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Union[str, Any]        =						np.take(self.block_records							,     indices=SCREAMING_SNAKE_CASE							,     axis=0		)
  
        UpperCamelCase__    :     Dict        =						self.tokenizer.decode(question_input_ids[0]							,     skip_special_tokens=SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     Dict        =						[]
        UpperCamelCase__    :     Tuple        =						[]
        for retrieved_block in retrieved_blocks:
             text.append(SCREAMING_SNAKE_CASE		)
             text_pair.append(retrieved_block.decode()		)
  
        UpperCamelCase__    :     List[str]        =						self.tokenizer(
            SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     padding=SCREAMING_SNAKE_CASE							,     truncation=SCREAMING_SNAKE_CASE							,     return_special_tokens_mask=SCREAMING_SNAKE_CASE							,     max_length=SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Dict        =						concat_inputs.convert_to_tensors(SCREAMING_SNAKE_CASE		)
  
        if answer_ids is not None:
             return self.block_has_answer(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		) + (concat_inputs_tensors,)
        else:
             return (None, None, None, concat_inputs_tensors)
  
  
  
  
  
  
  
   @classmethod
   def       __lowercase      (	cls      :     int							,     SCREAMING_SNAKE_CASE      :     Optional[Union[str, os.PathLike]]							,     *SCREAMING_SNAKE_CASE      :     Tuple							,     **SCREAMING_SNAKE_CASE      :     Union[str, Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        if os.path.isdir(SCREAMING_SNAKE_CASE		):
             UpperCamelCase__    :     Optional[Any]        =						os.path.join(SCREAMING_SNAKE_CASE							,     _REALM_BLOCK_RECORDS_FILENAME		)
        else:
             UpperCamelCase__    :     List[str]        =						hf_hub_download(
                 repo_id=SCREAMING_SNAKE_CASE							,     filename=_REALM_BLOCK_RECORDS_FILENAME							,     **SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Union[str, Any]        =						np.load(SCREAMING_SNAKE_CASE							,     allow_pickle=SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     List[Any]        =						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE							,     *SCREAMING_SNAKE_CASE							,     **SCREAMING_SNAKE_CASE		)
  
        return cls(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     int							,     SCREAMING_SNAKE_CASE      :     Tuple		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        np.save(os.path.join(SCREAMING_SNAKE_CASE							,     _REALM_BLOCK_RECORDS_FILENAME		)							,     self.block_records		)
        # save tokenizer
        self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     int							,     SCREAMING_SNAKE_CASE      :     List[str]							,     SCREAMING_SNAKE_CASE      :     Any		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Optional[int]        =						[]
        UpperCamelCase__    :     Optional[int]        =						[]
        UpperCamelCase__    :     Any        =						[]
        UpperCamelCase__    :     int        =						0
  
        for input_id in concat_inputs.input_ids:
             UpperCamelCase__    :     List[str]        =						input_id.tolist()
             # Check answers between two [SEP] tokens
             UpperCamelCase__    :     Optional[int]        =						input_id_list.index(self.tokenizer.sep_token_id		)
             UpperCamelCase__    :     str        =						first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id		)
  
             start_pos.append([]		)
             end_pos.append([]		)
             for answer in answer_ids:
                  for idx in range(first_sep_idx + 1							,     SCREAMING_SNAKE_CASE		):
                       if answer[0] == input_id_list[idx]:
                            if input_id_list[idx : idx + len(SCREAMING_SNAKE_CASE		)] == answer:
                                 start_pos[-1].append(SCREAMING_SNAKE_CASE		)
                                 end_pos[-1].append(idx + len(SCREAMING_SNAKE_CASE		) - 1		)
  
             if len(start_pos[-1]		) == 0:
                  has_answers.append(SCREAMING_SNAKE_CASE		)
             else:
                  has_answers.append(SCREAMING_SNAKE_CASE		)
                  if len(start_pos[-1]		) > max_answers:
                       UpperCamelCase__    :     str        =						len(start_pos[-1]		)
  
          # Pad -1 to max_answers
        for start_pos_, end_pos_ in zip(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		):
             if len(start_pos_		) < max_answers:
                  UpperCamelCase__    :     Dict        =						[-1] * (max_answers - len(start_pos_		))
                  start_pos_ += padded
                  end_pos_ += padded
        return has_answers, start_pos, end_pos
style_context: 
  
  
  
  
  
  
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
   print('''#################### AVAILABLE DEVICES ####################''')
   print(jax.devices())
   print('''###########################################################''')
  
   # setup for wandb sweep
   lowerCamelCase			:							str       =Args()
   lowerCamelCase			:							str       =wandb.init(project='''bigbird-natural-questions''', config=args.__dict__)
   lowerCamelCase			:							Any       =dict(logger.config)
   del wandb_args["batch_size"]
   lowerCamelCase			:							str       =replace(args, **wandb_args)
   lowerCamelCase			:							List[str]       =args.base_dir + '''-''' + wandb.run.id
   lowerCamelCase			:							str       =replace(args, base_dir=base_dir)
   print(args)
  
   lowerCamelCase			:							List[Any]       =load_dataset('''json''', data_files=args.tr_data_path)['''train''']
   lowerCamelCase			:							str       =load_dataset('''json''', data_files=args.val_data_path)['''train''']
  
   # drop extra batch for now
   lowerCamelCase			:							List[Any]       =range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
   lowerCamelCase			:							Union[str, Any]       =tr_dataset.shuffle().select(indices)
   lowerCamelCase			:							List[str]       =range(len(val_dataset) - len(val_dataset) % args.batch_size)
   lowerCamelCase			:							Dict       =val_dataset.shuffle().select(indices)
  
   if os.environ.get('''TRAIN_ON_SMALL''', '''false''') == "true":
    lowerCamelCase			:							Optional[int]       =tr_dataset.shuffle().select(range(80000))
    lowerCamelCase			:							List[Any]       =val_dataset.shuffle().select(range(8000))
  
   print(tr_dataset)
   print(val_dataset)
  
   lowerCamelCase			:							int       =FlaxBigBirdForNaturalQuestions.from_pretrained(
       args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
   )
   lowerCamelCase			:							Union[str, Any]       =BigBirdTokenizerFast.from_pretrained(args.model_id)
   lowerCamelCase			:							Dict       =DataCollator(pad_id=tokenizer.pad_token_id, max_length=4096)
  
   lowerCamelCase			:							List[str]       ={
       '''lr''': args.lr,
       '''init_lr''': args.init_lr,
       '''warmup_steps''': args.warmup_steps,
       '''num_train_steps''': args.max_epochs * (len(tr_dataset) // args.batch_size),
       '''weight_decay''': args.weight_decay,
   }
   lowerCamelCase       ,    lowerCamelCase			:							Tuple       =build_tx(**tx_args)
  
   lowerCamelCase			:							Optional[Any]       =Trainer(
       args=args,
       data_collator=data_collator,
       model_save_fn=model.save_pretrained,
       train_step_fn=train_step,
       val_step_fn=val_step,
       logger=logger,
       scheduler_fn=lr,
   )
  
   lowerCamelCase			:							Tuple       =None
   lowerCamelCase			:							str       =trainer.create_state(model, tx, num_train_steps=tx_args['''num_train_steps'''], ckpt_dir=ckpt_dir)
   try:
    trainer.train(state, tr_dataset, val_dataset)
   except KeyboardInterrupt:
    print('''Oooops; TRAINING STOPPED UNFORTUNATELY''')
  
   print('''SAVING WEIGHTS IN `final-weights`''')
   lowerCamelCase			:							str       =jax_utils.unreplicate(state.params)
   model.save_pretrained(os.path.join(args.base_dir, '''final-weights'''), params=params)