






"""simple docstring"""



import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


_lowerCAmelCase  :  List[Any]  							=			"""block_records.npy"""


_lowerCAmelCase  :  Optional[Any]  							=			logging.get_logger(__name__)


def   SCREAMING_SNAKE_CASE__					(     snake_case					:   str ,				snake_case					:   int							)->		Any:






     '''simple docstring'''



     import tensorflow.compat.va as tf

     UpperCAmelCase__	:						Union[str, Any]       =       tf.data.TFRecordDataset(lowercase__ ,				buffer_size=512 * 1024 * 1024							)
     UpperCAmelCase__	:						Any       =       blocks_dataset.batch(lowercase__ ,				drop_remainder=lowercase__							)
     UpperCAmelCase__	:						Optional[int]       =       next(blocks_dataset.take(1							).as_numpy_iterator()							)

     return np_record




class  lowerCAmelCase__ :
   def __init__(		self			:  Union[str, Any]       , snake_case__			:  Dict       , snake_case__			:  List[str]       , snake_case__			:  Optional[Any]=2       , snake_case__			:  int=1_0_0_0       , snake_case__			:  Optional[Any]=1_0_0       , snake_case__			:  Tuple=1_0_0_0_0_0       , ):


        '''simple docstring'''
        from scann.scann_ops.py.scann_ops_pybind import builder as Builder

        UpperCAmelCase__	:						Any       =       Builder(db=snake_case__       , num_neighbors=snake_case__       , distance_measure="dot_product"       )
        UpperCAmelCase__	:						Any       =       builder.tree(
            num_leaves=snake_case__       , num_leaves_to_search=snake_case__       , training_sample_size=snake_case__       )
        UpperCAmelCase__	:						Dict       =       builder.score_ah(dimensions_per_block=snake_case__       )

        UpperCAmelCase__	:						Union[str, Any]       =       builder.build()




   def 			__a       (		self			:  Optional[Any]       , snake_case__			:  int       ):


        '''simple docstring'''
        UpperCAmelCase__	:						int       =       self.searcher.search_batched(question_projection.detach().cpu()       )
        return retrieved_block_ids.astype("int64"       )




class  lowerCAmelCase__ :
   def __init__(		self			:  Any       , snake_case__			:  str       , snake_case__			:  List[Any]       ):


        '''simple docstring'''
        super().__init__()
        UpperCAmelCase__	:						Any       =       block_records
        UpperCAmelCase__	:						str       =       tokenizer
   def __call__(		self			:  Dict       , snake_case__			:  int       , snake_case__			:  Optional[int]       , snake_case__			:  str       , snake_case__			:  Optional[Any]=None       , snake_case__			:  List[Any]="pt"       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Optional[int]       =       np.take(self.block_records       , indices=snake_case__       , axis=0       )

        UpperCAmelCase__	:						Dict       =       self.tokenizer.decode(question_input_ids[0]       , skip_special_tokens=snake_case__       )

        UpperCAmelCase__	:						List[str]       =       []
        UpperCAmelCase__	:						Optional[Any]       =       []
        for retrieved_block in retrieved_blocks:
             text.append(snake_case__       )
             text_pair.append(retrieved_block.decode()       )

        UpperCAmelCase__	:						Union[str, Any]       =       self.tokenizer(
            snake_case__       , snake_case__       , padding=snake_case__       , truncation=snake_case__       , return_special_tokens_mask=snake_case__       , max_length=snake_case__       )
        UpperCAmelCase__	:						Optional[int]       =       concat_inputs.convert_to_tensors(snake_case__       )

        if answer_ids is not None:
             return self.block_has_answer(snake_case__       , snake_case__       ) + (concat_inputs_tensors,)
        else:
             return (None, None, None, concat_inputs_tensors)
   @classmethod
   def 			__a       (		cls			:  Optional[int]       , snake_case__			:  List[str]       , *snake_case__			:  List[str]       , **snake_case__			:  Any       ):


        '''simple docstring'''







        if os.path.isdir(snake_case__       ):
             UpperCAmelCase__	:						Optional[Any]       =       os.path.join(snake_case__       , _REALM_BLOCK_RECORDS_FILENAME       )
        else:
             UpperCAmelCase__	:						str       =       hf_hub_download(
                 repo_id=snake_case__       , filename=_REALM_BLOCK_RECORDS_FILENAME       , **snake_case__       )
        UpperCAmelCase__	:						Optional[int]       =       np.load(snake_case__       , allow_pickle=snake_case__       )

        UpperCAmelCase__	:						Union[str, Any]       =       AutoTokenizer.from_pretrained(snake_case__       , *snake_case__       , **snake_case__       )

        return cls(snake_case__       , snake_case__       )
   def 			__a       (		self			:  Any       , snake_case__			:  Dict       ):


        '''simple docstring'''
        np.save(os.path.join(snake_case__       , _REALM_BLOCK_RECORDS_FILENAME       )       , self.block_records       )
        # save tokenizer
        self.tokenizer.save_pretrained(snake_case__       )




   def 			__a       (		self			:  Optional[Any]       , snake_case__			:  Optional[Any]       , snake_case__			:  Optional[int]       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Any       =       []
        UpperCAmelCase__	:						str       =       []
        UpperCAmelCase__	:						Any       =       []
        UpperCAmelCase__	:						List[Any]       =       0

        for input_id in concat_inputs.input_ids:
             UpperCAmelCase__	:						Union[str, Any]       =       input_id.tolist()
             # Check answers between two [SEP] tokens
             UpperCAmelCase__	:						Union[str, Any]       =       input_id_list.index(self.tokenizer.sep_token_id       )
             UpperCAmelCase__	:						Optional[int]       =       first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id       )

             start_pos.append([]       )
             end_pos.append([]       )
             for answer in answer_ids:
                  for idx in range(first_sep_idx + 1       , snake_case__       ):
                       if answer[0] == input_id_list[idx]:
                            if input_id_list[idx : idx + len(snake_case__       )] == answer:
                                 start_pos[-1].append(snake_case__       )
                                 end_pos[-1].append(idx + len(snake_case__       ) - 1       )

             if len(start_pos[-1]       ) == 0:
                  has_answers.append(snake_case__       )
             else:
                  has_answers.append(snake_case__       )
                  if len(start_pos[-1]       ) > max_answers:
                       UpperCAmelCase__	:						List[str]       =       len(start_pos[-1]       )

        # Pad -1 to max_answers
        for start_pos_, end_pos_ in zip(snake_case__       , snake_case__       ):
             if len(start_pos_       ) < max_answers:
                  UpperCAmelCase__	:						List[str]       =       [-1] * (max_answers - len(start_pos_       ))
                  start_pos_ += padded
                  end_pos_ += padded
        return has_answers, start_pos, end_pos










"""simple docstring"""



# Lint as: python3

import hashlib
from typing import Union


def   SCREAMING_SNAKE_CASE__					(     snake_case					:   Union[str, int, bytes]							)->		bytes:






     '''simple docstring'''



     if isinstance(snake_case ,				snake_case							):
          # Data already in bytes, returns as it as
          return hash_data
     elif isinstance(snake_case ,				snake_case							):
          # We keep the data as it as for it ot be later encoded to UTF-8
          # However replace `\\` with `/` for Windows compatibility
          UpperCAmelCase__	:						Union[str, Any]       =       hash_data.replace("\\" ,				"/"							)
     elif isinstance(snake_case ,				snake_case							):
          UpperCAmelCase__	:						int       =       str(snake_case							)
     else:
          # If data is not of the required type, raise error
          raise InvalidKeyError(snake_case							)

     return hash_data.encode("utf-8"							)




class  lowerCAmelCase__ (						__magic_name__      ):
   def __init__(		self			:  List[str]       , snake_case__			:  Union[str, Any]       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Tuple       =       "\nFAILURE TO GENERATE DATASET: Invalid key type detected"
        UpperCAmelCase__	:						str       =       f'\nFound Key {hash_data} of type {type(snake_case__       )}'
        UpperCAmelCase__	:						Union[str, Any]       =       "\nKeys should be either str, int or bytes type"
        super().__init__(f'{self.prefix}{self.err_msg}{self.suffix}'       )




class  lowerCAmelCase__ (						__magic_name__      ):
   def __init__(		self			:  List[str]       , snake_case__			:  Dict       , snake_case__			:  Optional[int]       , snake_case__			:  List[str]=""       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Union[str, Any]       =       key
        UpperCAmelCase__	:						Optional[Any]       =       duplicate_key_indices
        UpperCAmelCase__	:						str       =       fix_msg
        UpperCAmelCase__	:						Any       =       "Found multiple examples generated with the same key"
        if len(snake_case__       ) <= 2_0:
             UpperCAmelCase__	:						int       =       f'\nThe examples at index {", ".join(snake_case__       )} have the key {key}'
        else:
             UpperCAmelCase__	:						str       =       f'\nThe examples at index {", ".join(duplicate_key_indices[:2_0]       )}... ({len(snake_case__       ) - 2_0} more) have the key {key}'
        UpperCAmelCase__	:						Any       =       "\n" + fix_msg if fix_msg else ""
        super().__init__(f'{self.prefix}{self.err_msg}{self.suffix}'       )




class  lowerCAmelCase__ :
   def __init__(		self			:  Dict       , snake_case__			:  str       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Union[str, Any]       =       hashlib.mda(_as_bytes(snake_case__       )       )




   def 			__a       (		self			:  int       , snake_case__			:  Union[str, int, bytes]       ):


        '''simple docstring'''
        UpperCAmelCase__	:						int       =       self._split_mda.copy()
        UpperCAmelCase__	:						List[str]       =       _as_bytes(snake_case__       )
        mda.update(snake_case__       )
        # Convert to integer with hexadecimal conversion
        return int(mda.hexdigest()       , 1_6       )


