




import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__magic_name__     :     int  							=					"""block_records.npy"""


__magic_name__     :     Optional[int]  							=					logging.get_logger(__name__)
def 		a_       (   __lowerCAmelCase      ,   __lowerCAmelCase							):
  import tensorflow.compat.va as tf

  lowerCAmelCase__								=			tf.data.TFRecordDataset(__snake_case      ,   buffer_size=5_12 * 10_24 * 10_24							)
  lowerCAmelCase__								=			blocks_dataset.batch(__snake_case      ,   drop_remainder=__snake_case							)
  lowerCAmelCase__								=			next(blocks_dataset.take(1							).as_numpy_iterator()							)

  return np_record





class 						SCREAMING_SNAKE_CASE__		:
     def __init__(	self  :		Optional[int]					,	__lowerCamelCase  :		Optional[Any]					,	__lowerCamelCase  :		Tuple					,	__lowerCamelCase  :		Any=2					,	__lowerCamelCase  :		List[Any]=10_00					,	__lowerCamelCase  :		Optional[Any]=1_00					,	__lowerCamelCase  :		List[str]=10_00_00					,	):



       """simple docstring"""
       from scann.scann_ops.py.scann_ops_pybind import builder as Builder

       lowerCAmelCase__								=			Builder(db=__lowerCamelCase					,	num_neighbors=__lowerCamelCase					,	distance_measure='''dot_product'''	)
       lowerCAmelCase__								=			builder.tree(
           num_leaves=__lowerCamelCase					,	num_leaves_to_search=__lowerCamelCase					,	training_sample_size=__lowerCamelCase	)
       lowerCAmelCase__								=			builder.score_ah(dimensions_per_block=__lowerCamelCase	)

       lowerCAmelCase__								=			builder.build()



     def 					A__			(	self  :		int					,	__lowerCamelCase  :		Tuple	):



       """simple docstring"""
       lowerCAmelCase__							,			lowerCAmelCase__								=			self.searcher.search_batched(question_projection.detach().cpu()	)
       return retrieved_block_ids.astype('''int64'''	)





class 						SCREAMING_SNAKE_CASE__		:
     def __init__(	self  :		List[str]					,	__lowerCamelCase  :		Dict					,	__lowerCamelCase  :		str	):



       """simple docstring"""
       super().__init__()
       lowerCAmelCase__								=			block_records
       lowerCAmelCase__								=			tokenizer
     def __call__(	self  :		List[Any]					,	__lowerCamelCase  :		List[Any]					,	__lowerCamelCase  :		Optional[int]					,	__lowerCamelCase  :		Dict					,	__lowerCamelCase  :		Dict=None					,	__lowerCamelCase  :		Optional[int]="pt"	):



       """simple docstring"""
       lowerCAmelCase__								=			np.take(self.block_records					,	indices=__lowerCamelCase					,	axis=0	)

       lowerCAmelCase__								=			self.tokenizer.decode(question_input_ids[0]					,	skip_special_tokens=__lowerCamelCase	)

       lowerCAmelCase__								=			[]
       lowerCAmelCase__								=			[]
       for retrieved_block in retrieved_blocks:
         text.append(__lowerCamelCase	)
         text_pair.append(retrieved_block.decode()	)

       lowerCAmelCase__								=			self.tokenizer(
           __lowerCamelCase					,	__lowerCamelCase					,	padding=__lowerCamelCase					,	truncation=__lowerCamelCase					,	return_special_tokens_mask=__lowerCamelCase					,	max_length=__lowerCamelCase	)
       lowerCAmelCase__								=			concat_inputs.convert_to_tensors(__lowerCamelCase	)

       if answer_ids is not None:
         return self.block_has_answer(__lowerCamelCase					,	__lowerCamelCase	) + (concat_inputs_tensors,)
       else:
         return (None, None, None, concat_inputs_tensors)
     @classmethod
     def 					A__			(	cls  :		Optional[Any]					,	__lowerCamelCase  :		Tuple					,	*__lowerCamelCase  :		Any					,	**__lowerCamelCase  :		Optional[Any]	):



       """simple docstring"""







       if os.path.isdir(__lowerCamelCase	):
         lowerCAmelCase__								=			os.path.join(__lowerCamelCase					,	_REALM_BLOCK_RECORDS_FILENAME	)
       else:
         lowerCAmelCase__								=			hf_hub_download(
             repo_id=__lowerCamelCase					,	filename=_REALM_BLOCK_RECORDS_FILENAME					,	**__lowerCamelCase	)
       lowerCAmelCase__								=			np.load(__lowerCamelCase					,	allow_pickle=__lowerCamelCase	)

       lowerCAmelCase__								=			AutoTokenizer.from_pretrained(__lowerCamelCase					,	*__lowerCamelCase					,	**__lowerCamelCase	)

       return cls(__lowerCamelCase					,	__lowerCamelCase	)
     def 					A__			(	self  :		Optional[int]					,	__lowerCamelCase  :		Tuple	):



       """simple docstring"""
       np.save(os.path.join(__lowerCamelCase					,	_REALM_BLOCK_RECORDS_FILENAME	)					,	self.block_records	)
       # save tokenizer
       self.tokenizer.save_pretrained(__lowerCamelCase	)



     def 					A__			(	self  :		Dict					,	__lowerCamelCase  :		List[Any]					,	__lowerCamelCase  :		List[Any]	):



       """simple docstring"""
       lowerCAmelCase__								=			[]
       lowerCAmelCase__								=			[]
       lowerCAmelCase__								=			[]
       lowerCAmelCase__								=			0

       for input_id in concat_inputs.input_ids:
         lowerCAmelCase__								=			input_id.tolist()
         # Check answers between two [SEP] tokens
         lowerCAmelCase__								=			input_id_list.index(self.tokenizer.sep_token_id	)
         lowerCAmelCase__								=			first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id	)

         start_pos.append([]	)
         end_pos.append([]	)
         for answer in answer_ids:
           for idx in range(first_sep_idx + 1					,	__lowerCamelCase	):
             if answer[0] == input_id_list[idx]:
               if input_id_list[idx : idx + len(__lowerCamelCase	)] == answer:
                 start_pos[-1].append(__lowerCamelCase	)
                 end_pos[-1].append(idx + len(__lowerCamelCase	) - 1	)

         if len(start_pos[-1]	) == 0:
           has_answers.append(__lowerCamelCase	)
         else:
           has_answers.append(__lowerCamelCase	)
           if len(start_pos[-1]	) > max_answers:
             lowerCAmelCase__								=			len(start_pos[-1]	)

            # Pad -1 to max_answers
       for start_pos_, end_pos_ in zip(__lowerCamelCase					,	__lowerCamelCase	):
         if len(start_pos_	) < max_answers:
           lowerCAmelCase__								=			[-1] * (max_answers - len(start_pos_	))
           start_pos_ += padded
           end_pos_ += padded
       return has_answers, start_pos, end_pos








import json

import requests

from .fetch_github_info import AUTHENTICATED_USER_ENDPOINT, fetch_github_info
def 		a_       (   __lowerCAmelCase							):





  class 						SCREAMING_SNAKE_CASE__		:
       def __init__(	self  :		List[str]					,	__lowerCamelCase  :		Any	):



         """simple docstring"""
         assert isinstance(__lowerCamelCase					,	(bytes, str)	)
         lowerCAmelCase__								=			content



       def 					A__			(	self  :		int	):



         """simple docstring"""
         return json.loads(self.content	)

  def mock_response(*__lowerCAmelCase      ,   **__lowerCAmelCase							):
    assert args[0] == AUTHENTICATED_USER_ENDPOINT
    assert "Authorization" in kwargs["headers"]
    assert kwargs["headers"]["Authorization"].startswith('''token '''							)
    assert "Accept" in kwargs["headers"]
    return FakeResponse(B'''{"login":"test","id":1}'''							)

  monkeypatch.setattr(__lowerCAmelCase      ,   '''get'''      ,   __lowerCAmelCase							)
  lowerCAmelCase__								=			fetch_github_info('''token'''							)
  assert result["login"] == "test"
  assert result["id"] == 1

