code: 
  
  
  import os
  from typing import Optional, Union
  
  import numpy as np
  from huggingface_hub import hf_hub_download
  
  from ... import AutoTokenizer
  from ...utils import logging
  
  
  A_    :					Optional[int]             ="""block_records.npy"""
  
  
  A_    :					str             =logging.get_logger(__name__)
  
  
  
  
  def   lowerCamelCase_       (					UpperCAmelCase__       ,    UpperCAmelCase__		):
  
  
  
  						"""simple docstring"""
  
  
  
  
  						import tensorflow.compat.va as tf
  
  						a_					=		tf.data.TFRecordDataset(UpperCAmelCase__       ,    buffer_size=512 * 1_024 * 1_024		)
  						a_					=		blocks_dataset.batch(UpperCAmelCase__       ,    drop_remainder=UpperCAmelCase__		)
  						a_					=		next(blocks_dataset.take(1		).as_numpy_iterator()		)
  
  						return np_record
  
  
  
  
  
  class        lowercase_				:
  
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  
  
  							def __init__(   self			,    _UpperCAmelCase			,    _UpperCAmelCase			,    _UpperCAmelCase=2			,    _UpperCAmelCase=1_000			,    _UpperCAmelCase=100			,    _UpperCAmelCase=100_000			,    ):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													from scann.scann_ops.py.scann_ops_pybind import builder as Builder
  
  													a_					=		Builder(db=_UpperCAmelCase			,    num_neighbors=_UpperCAmelCase			,    distance_measure="""dot_product"""		)
  													a_					=		builder.tree(
  													    num_leaves=_UpperCAmelCase			,    num_leaves_to_search=_UpperCAmelCase			,    training_sample_size=_UpperCAmelCase		)
  													a_					=		builder.score_ah(dimensions_per_block=_UpperCAmelCase		)
  
  													a_					=		builder.build()
  
  
  
  
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_			,   a_					=		self.searcher.search_batched(question_projection.detach().cpu()		)
  													return retrieved_block_ids.astype("""int64"""		)
  
  
  
  
  
  class        lowercase_				:
  
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  
  
  							def __init__(   self			,    _UpperCAmelCase			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													super().__init__()
  													a_					=		block_records
  													a_					=		tokenizer
  
  
  							def __call__(   self			,    _UpperCAmelCase			,    _UpperCAmelCase			,    _UpperCAmelCase			,    _UpperCAmelCase=None			,    _UpperCAmelCase="pt"		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		np.take(self.block_records			,    indices=_UpperCAmelCase			,    axis=0		)
  
  													a_					=		self.tokenizer.decode(question_input_ids[0]			,    skip_special_tokens=_UpperCAmelCase		)
  
  													a_					=		[]
  													a_					=		[]
  													for retrieved_block in retrieved_blocks:
  																			text.append(_UpperCAmelCase		)
  																			text_pair.append(retrieved_block.decode()		)
  
  													a_					=		self.tokenizer(
  													    _UpperCAmelCase			,    _UpperCAmelCase			,    padding=_UpperCAmelCase			,    truncation=_UpperCAmelCase			,    return_special_tokens_mask=_UpperCAmelCase			,    max_length=_UpperCAmelCase		)
  													a_					=		concat_inputs.convert_to_tensors(_UpperCAmelCase		)
  
  													if answer_ids is not None:
  																			return self.block_has_answer(_UpperCAmelCase			,    _UpperCAmelCase		) + (concat_inputs_tensors,)
  													else:
  																			return (None, None, None, concat_inputs_tensors)
  
  
  							@classmethod
  							def 						lowercase__     (   cls			,    _UpperCAmelCase			,    *_UpperCAmelCase			,    **_UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													if os.path.isdir(_UpperCAmelCase		):
  																			a_					=		os.path.join(_UpperCAmelCase			,    _REALM_BLOCK_RECORDS_FILENAME		)
  													else:
  																			a_					=		hf_hub_download(
  																			    repo_id=_UpperCAmelCase			,    filename=_REALM_BLOCK_RECORDS_FILENAME			,    **_UpperCAmelCase		)
  													a_					=		np.load(_UpperCAmelCase			,    allow_pickle=_UpperCAmelCase		)
  
  													a_					=		AutoTokenizer.from_pretrained(_UpperCAmelCase			,    *_UpperCAmelCase			,    **_UpperCAmelCase		)
  
  													return cls(_UpperCAmelCase			,    _UpperCAmelCase		)
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													np.save(os.path.join(_UpperCAmelCase			,    _REALM_BLOCK_RECORDS_FILENAME		)			,    self.block_records		)
  													# save tokenizer
  													self.tokenizer.save_pretrained(_UpperCAmelCase		)
  
  
  
  
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		[]
  													a_					=		[]
  													a_					=		[]
  													a_					=		0
  
  													for input_id in concat_inputs.input_ids:
  																			a_					=		input_id.tolist()
  																			# Check answers between two [SEP] tokens
  																			a_					=		input_id_list.index(self.tokenizer.sep_token_id		)
  																			a_					=		first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id		)
  
  																			start_pos.append([]		)
  																			end_pos.append([]		)
  																			for answer in answer_ids:
  																									for idx in range(first_sep_idx + 1			,    _UpperCAmelCase		):
  																															if answer[0] == input_id_list[idx]:
  																																					if input_id_list[idx : idx + len(_UpperCAmelCase		)] == answer:
  																																											start_pos[-1].append(_UpperCAmelCase		)
  																																											end_pos[-1].append(idx + len(_UpperCAmelCase		) - 1		)
  
  																			if len(start_pos[-1]		) == 0:
  																									has_answers.append(_UpperCAmelCase		)
  																			else:
  																									has_answers.append(_UpperCAmelCase		)
  																									if len(start_pos[-1]		) > max_answers:
  																															a_					=		len(start_pos[-1]		)
  
          # Pad -1 to max_answers
  													for start_pos_, end_pos_ in zip(_UpperCAmelCase			,    _UpperCAmelCase		):
  																			if len(start_pos_		) < max_answers:
  																									a_					=		[-1] * (max_answers - len(start_pos_		))
  																									start_pos_ += padded
  																									end_pos_ += padded
  													return has_answers, start_pos, end_pos
style_context: 
  
  
  import importlib
  import sys
  from argparse import REMAINDER, ArgumentParser
  from pathlib import Path
  
  import torch_xla.distributed.xla_multiprocessing as xmp
  
  
  
  
  def   lowerCamelCase_       (					):
  
  
  
  						"""simple docstring"""
  
  
  
  
  						a_					=		ArgumentParser(
  						    description=(
  						        """PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"""
  						    )      )
  
  						# Optional arguments for the launch helper
  						parser.add_argument("""--num_cores"""       ,    type=UpperCAmelCase__       ,    default=1       ,    help="""Number of TPU cores to use (1 or 8)."""      )
  
  						# positional
  						parser.add_argument(
  						    """training_script"""       ,    type=UpperCAmelCase__       ,    help=(
  						        """The full path to the single TPU training """
  						        """program/script to be launched in parallel, """
  						        """followed by all the arguments for the """
  						        """training script"""
  						    )       ,    )
  
  						# rest from the training program
  						parser.add_argument("""training_script_args"""       ,    nargs=UpperCAmelCase__      )
  
  						return parser.parse_args()
  
  
  
  
  def   lowerCamelCase_       (					):
  
  
  
  						"""simple docstring"""
  
  
  
  
  						a_					=		parse_args()
  
  						# Import training_script as a module.
  						a_					=		Path(args.training_script      )
  						sys.path.append(str(script_fpath.parent.resolve()      )      )
  						a_					=		script_fpath.stem
  						a_					=		importlib.import_module(UpperCAmelCase__      )
  
  						# Patch sys.argv
  						a_					=		[args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores      )]
  
  						xmp.spawn(mod._mp_fn       ,    args=()       ,    nprocs=args.num_cores      )
  
  
  if __name__ == "__main__":
  	main()