


'''simple docstring'''

import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__lowercase:		Dict    						=  'block_records.npy'


__lowercase:		Optional[int]    						=  logging.get_logger(__name__)



def 							SCREAMING_SNAKE_CASE__(		_UpperCamelCase   :					List[Any]     ,		_UpperCamelCase   :					Dict )		->       Any:





   '''simple docstring'''







   import tensorflow.compat.va as tf

   UpperCamelCase__			    =    tf.data.TFRecordDataset(snake_case__     ,		buffer_size=5_12 * 10_24 * 10_24 )
   UpperCamelCase__			    =    blocks_dataset.batch(snake_case__     ,		drop_remainder=snake_case__ )
   UpperCamelCase__			    =    next(blocks_dataset.take(1 ).as_numpy_iterator() )

   return np_record







class 			UpperCAmelCase			:

 def __init__(			self			:						Optional[int],       a_			:						Optional[Any],       a_			:						int,       a_			:						str=2,       a_			:						List[str]=1000,       a_			:						Any=100,       a_			:						Union[str, Any]=10_0000,       ):





    """simple docstring"""



    from scann.scann_ops.py.scann_ops_pybind import builder as Builder

    UpperCamelCase__			    =    Builder(db=lowercase__,       num_neighbors=lowercase__,       distance_measure="dot_product"       )
    UpperCamelCase__			    =    builder.tree(
        num_leaves=lowercase__,       num_leaves_to_search=lowercase__,       training_sample_size=lowercase__       )
    UpperCamelCase__			    =    builder.score_ah(dimensions_per_block=lowercase__       )

    UpperCamelCase__			    =    builder.build()

 def 		lowercase_    (			self			:						Optional[int],       a_			:						Tuple       ):





    """simple docstring"""



    UpperCamelCase__    ,		UpperCamelCase__			    =    self.searcher.search_batched(question_projection.detach().cpu()       )
    return retrieved_block_ids.astype("int64"       )







class 			UpperCAmelCase			:

 def __init__(			self			:						Union[str, Any],       a_			:						Dict,       a_			:						Optional[Any]       ):





    """simple docstring"""



    super().__init__()
    UpperCamelCase__			    =    block_records
    UpperCamelCase__			    =    tokenizer

 def __call__(			self			:						Dict,       a_			:						Union[str, Any],       a_			:						str,       a_			:						Tuple,       a_			:						Tuple=None,       a_			:						Optional[int]="pt"       ):





    """simple docstring"""



    UpperCamelCase__			    =    np.take(self.block_records,       indices=lowercase__,       axis=0       )

    UpperCamelCase__			    =    self.tokenizer.decode(question_input_ids[0],       skip_special_tokens=lowercase__       )

    UpperCamelCase__			    =    []
    UpperCamelCase__			    =    []
    for retrieved_block in retrieved_blocks:
       text.append(lowercase__       )
       text_pair.append(retrieved_block.decode()       )

    UpperCamelCase__			    =    self.tokenizer(
        lowercase__,       lowercase__,       padding=lowercase__,       truncation=lowercase__,       return_special_tokens_mask=lowercase__,       max_length=lowercase__       )
    UpperCamelCase__			    =    concat_inputs.convert_to_tensors(lowercase__       )

    if answer_ids is not None:
       return self.block_has_answer(lowercase__,       lowercase__       ) + (concat_inputs_tensors,)
    else:
       return (None, None, None, concat_inputs_tensors)

 @classmethod
 def 		lowercase_    (			cls			:						List[str],       a_			:						Optional[Union[str, os.PathLike]],       *a_			:						str,       **a_			:						Union[str, Any]       ):





    """simple docstring"""



    if os.path.isdir(lowercase__       ):
       UpperCamelCase__			    =    os.path.join(lowercase__,       _REALM_BLOCK_RECORDS_FILENAME       )
    else:
       UpperCamelCase__			    =    hf_hub_download(
           repo_id=lowercase__,       filename=_REALM_BLOCK_RECORDS_FILENAME,       **lowercase__       )
    UpperCamelCase__			    =    np.load(lowercase__,       allow_pickle=lowercase__       )

    UpperCamelCase__			    =    AutoTokenizer.from_pretrained(lowercase__,       *lowercase__,       **lowercase__       )

    return cls(lowercase__,       lowercase__       )

 def 		lowercase_    (			self			:						List[str],       a_			:						List[str]       ):





    """simple docstring"""



    np.save(os.path.join(lowercase__,       _REALM_BLOCK_RECORDS_FILENAME       ),       self.block_records       )
    # save tokenizer
    self.tokenizer.save_pretrained(lowercase__       )

 def 		lowercase_    (			self			:						List[Any],       a_			:						Optional[int],       a_			:						str       ):





    """simple docstring"""



    UpperCamelCase__			    =    []
    UpperCamelCase__			    =    []
    UpperCamelCase__			    =    []
    UpperCamelCase__			    =    0

    for input_id in concat_inputs.input_ids:
       UpperCamelCase__			    =    input_id.tolist()
       # Check answers between two [SEP] tokens
       UpperCamelCase__			    =    input_id_list.index(self.tokenizer.sep_token_id       )
       UpperCamelCase__			    =    first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id       )

       start_pos.append([]       )
       end_pos.append([]       )
       for answer in answer_ids:
          for idx in range(first_sep_idx + 1,       lowercase__       ):
             if answer[0] == input_id_list[idx]:
                if input_id_list[idx : idx + len(lowercase__       )] == answer:
                   start_pos[-1].append(lowercase__       )
                   end_pos[-1].append(idx + len(lowercase__       ) - 1       )

       if len(start_pos[-1]       ) == 0:
          has_answers.append(lowercase__       )
       else:
          has_answers.append(lowercase__       )
          if len(start_pos[-1]       ) > max_answers:
             UpperCamelCase__			    =    len(start_pos[-1]       )

        # Pad -1 to max_answers
    for start_pos_, end_pos_ in zip(lowercase__,       lowercase__       ):
       if len(start_pos_       ) < max_answers:
          UpperCamelCase__			    =    [-1] * (max_answers - len(start_pos_       ))
          start_pos_ += padded
          end_pos_ += padded
    return has_answers, start_pos, end_pos







'''simple docstring'''







import sys
from collections.abc import Mapping

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter






class        a__     (      TensorFormatter[Mapping, np.ndarray, Mapping]    ):
     '''simple docstring'''







     def __init__(self     :						Tuple					,					lowercase__     :						Dict=None					,					**lowercase__     :						List[Any]       )	->							str:
           super().__init__(features=lowercase__       )
           __A		 =   np_array_kwargs






     def      _UpperCamelCase (self     :						int					,					lowercase__     :						Tuple       )	->							List[Any]:
           if isinstance(lowercase__					,					lowercase__       ):
                 if column and all(
                     isinstance(lowercase__					,					np.ndarray       ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column       ):
                       return np.stack(lowercase__       )
                 else:
                       # don't use np.array(column, dtype=object)
                       # since it fails in certain cases
                       # see https://stackoverflow.com/q/51005699
                       __A		 =   np.empty(len(lowercase__       )					,					dtype=lowercase__       )
                       __A		 =   column
                       return out
           return column






     def      _UpperCamelCase (self     :						Optional[Any]					,					lowercase__     :						Optional[int]       )	->							Tuple:
           if isinstance(lowercase__					,					(str, bytes, type(lowercase__       ))       ):
                 return value
           elif isinstance(lowercase__					,					(np.character, np.ndarray)       ) and np.issubdtype(value.dtype					,					np.character       ):
                 return value
           elif isinstance(lowercase__					,					np.number       ):
                 return value

           __A		 =   {}

           if isinstance(lowercase__					,					np.ndarray       ) and np.issubdtype(value.dtype					,					np.integer       ):
                 __A		 =   {"dtype": np.intaa}
           elif isinstance(lowercase__					,					np.ndarray       ) and np.issubdtype(value.dtype					,					np.floating       ):
                 __A		 =   {"dtype": np.floataa}
           elif config.PIL_AVAILABLE and "PIL" in sys.modules:
                 import PIL.Image

                 if isinstance(lowercase__					,					PIL.Image.Image       ):
                       return np.asarray(lowercase__					,					**self.np_array_kwargs       )

           return np.asarray(lowercase__					,					**{**default_dtype, **self.np_array_kwargs}       )






     def      _UpperCamelCase (self     :						List[Any]					,					lowercase__     :						str       )	->							Optional[Any]:
           # support for torch, tf, jax etc.
           if config.TORCH_AVAILABLE and "torch" in sys.modules:
                 import torch

                 if isinstance(lowercase__					,					torch.Tensor       ):
                       return self._tensorize(data_struct.detach().cpu().numpy()[()]       )
           if hasattr(lowercase__					,					"__array__"       ) and not isinstance(lowercase__					,					(np.ndarray, np.character, np.number)       ):
                 __A		 =   data_struct.__array__()
           # support for nested types like struct of list of struct
           if isinstance(lowercase__					,					np.ndarray       ):
                 if data_struct.dtype == object:
                       return self._consolidate([self.recursive_tensorize(lowercase__       ) for substruct in data_struct]       )
           if isinstance(lowercase__					,					(list, tuple)       ):
                 return self._consolidate([self.recursive_tensorize(lowercase__       ) for substruct in data_struct]       )
           return self._tensorize(lowercase__       )






     def      _UpperCamelCase (self     :						Optional[Any]					,					lowercase__     :						dict       )	->							Dict:
           return map_nested(self._recursive_tensorize					,					lowercase__					,					map_list=lowercase__       )






     def      _UpperCamelCase (self     :						Optional[int]					,					lowercase__     :						pa.Table       )	->							Mapping:
           __A		 =   self.numpy_arrow_extractor().extract_row(lowercase__       )
           __A		 =   self.python_features_decoder.decode_row(lowercase__       )
           return self.recursive_tensorize(lowercase__       )






     def      _UpperCamelCase (self     :						List[Any]					,					lowercase__     :						pa.Table       )	->							np.ndarray:
           __A		 =   self.numpy_arrow_extractor().extract_column(lowercase__       )
           __A		 =   self.python_features_decoder.decode_column(lowercase__					,					pa_table.column_names[0]       )
           __A		 =   self.recursive_tensorize(lowercase__       )
           __A		 =   self._consolidate(lowercase__       )
           return column






     def      _UpperCamelCase (self     :						Optional[Any]					,					lowercase__     :						pa.Table       )	->							Mapping:
           __A		 =   self.numpy_arrow_extractor().extract_batch(lowercase__       )
           __A		 =   self.python_features_decoder.decode_batch(lowercase__       )
           __A		 =   self.recursive_tensorize(lowercase__       )
           for column_name in batch:
                 __A		 =   self._consolidate(batch[column_name]       )
           return batch


