





'''simple docstring'''






import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__UpperCamelCase      :   Optional[int]								=     '''block_records.npy'''


__UpperCamelCase      :   Optional[int]								=     logging.get_logger(__name__)





def      lowercase  (  lowerCAmelCase :						str   , lowerCAmelCase :						int):



				"""simple docstring"""


				import tensorflow.compat.va as tf

				_A :				Optional[Any]				=  tf.data.TFRecordDataset(a_   , buffer_size=512 * 1024 * 1024)
				_A :				List[str]				=  blocks_dataset.batch(a_   , drop_remainder=a_)
				_A :				str				=  next(blocks_dataset.take(1).as_numpy_iterator())

				return np_record



class  lowerCamelCase__				:


	"""simple docstring"""





	def __init__(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__   ,     UpperCAmelCase__=2   ,     UpperCAmelCase__=1_0_0_0   ,     UpperCAmelCase__=1_0_0   ,     UpperCAmelCase__=1_0_0_0_0_0   ,     )       ->				str:
					from scann.scann_ops.py.scann_ops_pybind import builder as Builder

					_A :				Any				=  Builder(db=__UpperCamelCase   ,     num_neighbors=__UpperCamelCase   ,     distance_measure='''dot_product'''			)
					_A :				Union[str, Any]				=  builder.tree(
					    num_leaves=__UpperCamelCase   ,     num_leaves_to_search=__UpperCamelCase   ,     training_sample_size=__UpperCamelCase			)
					_A :				List[str]				=  builder.score_ah(dimensions_per_block=__UpperCamelCase			)

					_A :				Tuple				=  builder.build()




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__			)       ->				Any:
					_A	,       _A :				int				=  self.searcher.search_batched(question_projection.detach().cpu()			)
					return retrieved_block_ids.astype('''int64'''			)



class  lowerCamelCase__				:


	"""simple docstring"""





	def __init__(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__			)       ->				str:
					super().__init__()
					_A :				List[Any]				=  block_records
					_A :				Union[str, Any]				=  tokenizer




	def __call__(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__   ,     UpperCAmelCase__   ,     UpperCAmelCase__=None   ,     UpperCAmelCase__="pt"			)       ->				Optional[int]:
					_A :				List[Any]				=  np.take(self.block_records   ,     indices=__UpperCamelCase   ,     axis=0			)

					_A :				List[Any]				=  self.tokenizer.decode(question_input_ids[0]   ,     skip_special_tokens=__UpperCamelCase			)

					_A :				Union[str, Any]				=  []
					_A :				Union[str, Any]				=  []
					for retrieved_block in retrieved_blocks:
									text.append(__UpperCamelCase			)
									text_pair.append(retrieved_block.decode()			)

					_A :				List[str]				=  self.tokenizer(
					    __UpperCamelCase   ,     __UpperCamelCase   ,     padding=__UpperCamelCase   ,     truncation=__UpperCamelCase   ,     return_special_tokens_mask=__UpperCamelCase   ,     max_length=__UpperCamelCase			)
					_A :				Optional[int]				=  concat_inputs.convert_to_tensors(__UpperCamelCase			)

					if answer_ids is not None:
									return self.block_has_answer(__UpperCamelCase   ,     __UpperCamelCase			) + (concat_inputs_tensors,)
					else:
									return (None, None, None, concat_inputs_tensors)




	@classmethod
	def 				_lowerCamelCase				(					cls   ,     UpperCAmelCase__   ,     *UpperCAmelCase__   ,     **UpperCAmelCase__			)       ->				List[str]:





					if os.path.isdir(__UpperCamelCase			):
									_A :				Optional[int]				=  os.path.join(__UpperCamelCase   ,     _REALM_BLOCK_RECORDS_FILENAME			)
					else:
									_A :				Any				=  hf_hub_download(
									    repo_id=__UpperCamelCase   ,     filename=_REALM_BLOCK_RECORDS_FILENAME   ,     **__UpperCamelCase			)
					_A :				Dict				=  np.load(__UpperCamelCase   ,     allow_pickle=__UpperCamelCase			)

					_A :				Optional[Any]				=  AutoTokenizer.from_pretrained(__UpperCamelCase   ,     *__UpperCamelCase   ,     **__UpperCamelCase			)

					return cls(__UpperCamelCase   ,     __UpperCamelCase			)




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__			)       ->				Any:
					np.save(os.path.join(__UpperCamelCase   ,     _REALM_BLOCK_RECORDS_FILENAME			)   ,     self.block_records			)
					# save tokenizer
					self.tokenizer.save_pretrained(__UpperCamelCase			)




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__			)       ->				Dict:
					_A :				Optional[Any]				=  []
					_A :				Union[str, Any]				=  []
					_A :				Tuple				=  []
					_A :				Union[str, Any]				=  0

					for input_id in concat_inputs.input_ids:
									_A :				Any				=  input_id.tolist()
									# Check answers between two [SEP] tokens
									_A :				Optional[int]				=  input_id_list.index(self.tokenizer.sep_token_id			)
									_A :				Optional[Any]				=  first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id			)

									start_pos.append([]			)
									end_pos.append([]			)
									for answer in answer_ids:
													for idx in range(first_sep_idx + 1   ,     __UpperCamelCase			):
																	if answer[0] == input_id_list[idx]:
																					if input_id_list[idx : idx + len(__UpperCamelCase			)] == answer:
																									start_pos[-1].append(__UpperCamelCase			)
																									end_pos[-1].append(idx + len(__UpperCamelCase			) - 1			)

									if len(start_pos[-1]			) == 0:
													has_answers.append(__UpperCamelCase			)
									else:
													has_answers.append(__UpperCamelCase			)
													if len(start_pos[-1]			) > max_answers:
																	_A :				Any				=  len(start_pos[-1]			)

        # Pad -1 to max_answers
					for start_pos_, end_pos_ in zip(__UpperCamelCase   ,     __UpperCamelCase			):
									if len(start_pos_			) < max_answers:
													_A :				str				=  [-1] * (max_answers - len(start_pos_			))
													start_pos_ += padded
													end_pos_ += padded
					return has_answers, start_pos, end_pos



from hashlib import mda, shaaaa

A__							=				(shaaaa, mda)

class 					__UpperCamelCase					:







	def __init__( self:			Optional[int]	,			__UpperCamelCase:			int = 8   ):

						'''simple docstring'''





						__magic_name__				=     0b0
						__magic_name__				=     size







	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						__magic_name__				=     self.hash_(__UpperCamelCase   )
						self.bitarray |= h







	def 						_SCREAMING_SNAKE_CASE				( self:			List[str]	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						__magic_name__				=     self.hash_(__UpperCamelCase   )
						return (h & self.bitarray) == h







	def __contains__( self:			Optional[int]	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						return self.exists(__UpperCamelCase   )







	def 						_SCREAMING_SNAKE_CASE				( self:			int	,			__UpperCamelCase:			int   ):

						'''simple docstring'''





						__magic_name__				=     bin(__UpperCamelCase   )[2:]
						return res.zfill(self.size   )







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			str   ):

						'''simple docstring'''





						return self.format_bin(self.bitarray   )







	def 						_SCREAMING_SNAKE_CASE				( self:			str	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						__magic_name__				=     0b0
						for func in HASH_FUNCTIONS:
											__magic_name__				=     (
											    int.from_bytes(func(value.encode()   ).digest()	,			'little'   ) % self.size
											)
											res |= 2**position
						return res







	def 						_SCREAMING_SNAKE_CASE				( self:			Dict	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						return self.format_bin(self.hash_(__UpperCamelCase   )   )







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[int]   ):

						'''simple docstring'''





						__magic_name__				=     bin(self.bitarray   ).count('1'   )
						return (n_ones / self.size) ** len(__UpperCamelCase   )




