code: 
  
  
  import os
  from typing import Optional, Union
  
  import numpy as np
  from huggingface_hub import hf_hub_download
  
  from ... import AutoTokenizer
  from ...utils import logging
  
  
  _SCREAMING_SNAKE_CASE    :				int       =					"block_records.npy"
  
  
  _SCREAMING_SNAKE_CASE    :				Dict       =					logging.get_logger(__name__)
  def     UpperCAmelCase__		(UpperCamelCase_							,UpperCamelCase_       ):
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       import tensorflow.compat.va as tf
  
       snake_case					=       tf.data.TFRecordDataset(UpperCamelCase_							,buffer_size=5_12 * 10_24 * 10_24       )
       snake_case					=       blocks_dataset.batch(UpperCamelCase_							,drop_remainder=UpperCamelCase_       )
       snake_case					=       next(blocks_dataset.take(1       ).as_numpy_iterator()       )
  
       return np_record
  
  class       A__      :
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      __snake_case    ,      __snake_case    ,      __snake_case=2    ,      __snake_case=1_0_0_0    ,      __snake_case=1_0_0    ,      __snake_case=1_0_0_0_0_0    ,      ):
           from scann.scann_ops.py.scann_ops_pybind import builder as Builder
  
           snake_case					=       Builder(db=__snake_case    ,      num_neighbors=__snake_case    ,      distance_measure='''dot_product'''					)
           snake_case					=       builder.tree(
               num_leaves=__snake_case    ,      num_leaves_to_search=__snake_case    ,      training_sample_size=__snake_case					)
           snake_case					=       builder.score_ah(dimensions_per_block=__snake_case					)
  
           snake_case					=       builder.build()
  
  
  
  
      def 	a_    ( self    ,      __snake_case					):
           snake_case				,						snake_case					=       self.searcher.search_batched(question_projection.detach().cpu()					)
           return retrieved_block_ids.astype('''int64'''					)
  
  
  
  
  
  
  class       A__      :
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      __snake_case    ,      __snake_case					):
           super().__init__()
           snake_case					=       block_records
           snake_case					=       tokenizer
      def __call__( self    ,      __snake_case    ,      __snake_case    ,      __snake_case    ,      __snake_case=None    ,      __snake_case="pt"					):
           snake_case					=       np.take(self.block_records    ,      indices=__snake_case    ,      axis=0					)
  
           snake_case					=       self.tokenizer.decode(question_input_ids[0]    ,      skip_special_tokens=__snake_case					)
  
           snake_case					=       []
           snake_case					=       []
           for retrieved_block in retrieved_blocks:
                text.append(__snake_case					)
                text_pair.append(retrieved_block.decode()					)
  
           snake_case					=       self.tokenizer(
               __snake_case    ,      __snake_case    ,      padding=__snake_case    ,      truncation=__snake_case    ,      return_special_tokens_mask=__snake_case    ,      max_length=__snake_case					)
           snake_case					=       concat_inputs.convert_to_tensors(__snake_case					)
  
           if answer_ids is not None:
                return self.block_has_answer(__snake_case    ,      __snake_case					) + (concat_inputs_tensors,)
           else:
                return (None, None, None, concat_inputs_tensors)
      @classmethod
      def 	a_    ( cls    ,      __snake_case    ,      *__snake_case    ,      **__snake_case					):
           if os.path.isdir(__snake_case					):
                snake_case					=       os.path.join(__snake_case    ,      _REALM_BLOCK_RECORDS_FILENAME					)
           else:
                snake_case					=       hf_hub_download(
                    repo_id=__snake_case    ,      filename=_REALM_BLOCK_RECORDS_FILENAME    ,      **__snake_case					)
           snake_case					=       np.load(__snake_case    ,      allow_pickle=__snake_case					)
  
           snake_case					=       AutoTokenizer.from_pretrained(__snake_case    ,      *__snake_case    ,      **__snake_case					)
  
           return cls(__snake_case    ,      __snake_case					)
      def 	a_    ( self    ,      __snake_case					):
           # save block records
           np.save(os.path.join(__snake_case    ,      _REALM_BLOCK_RECORDS_FILENAME					)    ,      self.block_records					)
           # save tokenizer
           self.tokenizer.save_pretrained(__snake_case					)
  
  
  
  
      def 	a_    ( self    ,      __snake_case    ,      __snake_case					):
           snake_case					=       []
           snake_case					=       []
           snake_case					=       []
           snake_case					=       0
  
           for input_id in concat_inputs.input_ids:
                snake_case					=       input_id.tolist()
                # Check answers between two [SEP] tokens
                snake_case					=       input_id_list.index(self.tokenizer.sep_token_id					)
                snake_case					=       first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id					)
  
                start_pos.append([]					)
                end_pos.append([]					)
                for answer in answer_ids:
                     for idx in range(first_sep_idx + 1    ,      __snake_case					):
                          if answer[0] == input_id_list[idx]:
                               if input_id_list[idx : idx + len(__snake_case					)] == answer:
                                    start_pos[-1].append(__snake_case					)
                                    end_pos[-1].append(idx + len(__snake_case					) - 1					)
  
                if len(start_pos[-1]					) == 0:
                     has_answers.append(__snake_case					)
                else:
                     has_answers.append(__snake_case					)
                     if len(start_pos[-1]					) > max_answers:
                          snake_case					=       len(start_pos[-1]					)
  
          # Pad -1 to max_answers
           for start_pos_, end_pos_ in zip(__snake_case    ,      __snake_case					):
                if len(start_pos_					) < max_answers:
                     snake_case					=       [-1] * (max_answers - len(start_pos_					))
                     start_pos_ += padded
                     end_pos_ += padded
           return has_answers, start_pos, end_pos
  
  
  
  
  
style_context: 
  
  
  from ..utils import deprecate
  from .attention_processor import (  # noqa: F401
      Attention,
      AttentionProcessor,
      AttnAddedKVProcessor,
      AttnProcessor2_0,
      LoRAAttnProcessor,
      LoRALinearLayer,
      LoRAXFormersAttnProcessor,
      SlicedAttnAddedKVProcessor,
      SlicedAttnProcessor,
      XFormersAttnProcessor,
  )
  from .attention_processor import AttnProcessor as AttnProcessorRename  # noqa: F401
  
  
  deprecate(
      "cross_attention",
      "0.20.0",
      "Importing from cross_attention is deprecated. Please import from diffusers.models.attention_processor instead.",
      standard_warn=False,
  )
  
  
  _SCREAMING_SNAKE_CASE    :				Optional[int]       =					AttentionProcessor
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      *__snake_case    ,      **__snake_case					):
           snake_case					=       F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"					)					)} instead.'''
           deprecate('''cross_attention'''    ,      '''0.20.0'''    ,      __snake_case    ,      standard_warn=__snake_case					)
           super().__init__(*__snake_case    ,      **__snake_case					)
  
  
  
  
  
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      *__snake_case    ,      **__snake_case					):
           snake_case					=       F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"					)					)} instead.'''
           deprecate('''cross_attention'''    ,      '''0.20.0'''    ,      __snake_case    ,      standard_warn=__snake_case					)
           super().__init__(*__snake_case    ,      **__snake_case					)
  
  
  
  
  
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      *__snake_case    ,      **__snake_case					):
           snake_case					=       F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"					)					)} instead.'''
           deprecate('''cross_attention'''    ,      '''0.20.0'''    ,      __snake_case    ,      standard_warn=__snake_case					)
           super().__init__(*__snake_case    ,      **__snake_case					)
  
  
  
  
  
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      *__snake_case    ,      **__snake_case					):
           snake_case					=       F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"					)					)} instead.'''
           deprecate('''cross_attention'''    ,      '''0.20.0'''    ,      __snake_case    ,      standard_warn=__snake_case					)
           super().__init__(*__snake_case    ,      **__snake_case					)
  
  
  
  
  
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      *__snake_case    ,      **__snake_case					):
           snake_case					=       F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"					)					)} instead.'''
           deprecate('''cross_attention'''    ,      '''0.20.0'''    ,      __snake_case    ,      standard_warn=__snake_case					)
           super().__init__(*__snake_case    ,      **__snake_case					)
  
  
  
  
  
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      *__snake_case    ,      **__snake_case					):
           snake_case					=       F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"					)					)} instead.'''
           deprecate('''cross_attention'''    ,      '''0.20.0'''    ,      __snake_case    ,      standard_warn=__snake_case					)
           super().__init__(*__snake_case    ,      **__snake_case					)
  
  
  
  
  
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      *__snake_case    ,      **__snake_case					):
           snake_case					=       F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"					)					)} instead.'''
           deprecate('''cross_attention'''    ,      '''0.20.0'''    ,      __snake_case    ,      standard_warn=__snake_case					)
           super().__init__(*__snake_case    ,      **__snake_case					)
  
  
  
  
  
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __init__( self    ,      *__snake_case    ,      **__snake_case					):
           snake_case					=       F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"					)					)} instead.'''
           deprecate('''cross_attention'''    ,      '''0.20.0'''    ,      __snake_case    ,      standard_warn=__snake_case					)
           super().__init__(*__snake_case    ,      **__snake_case					)
  
  
  
  
  