code: 
  
  
  
  import os
  from typing import Optional, Union
  
  import numpy as np
  from huggingface_hub import hf_hub_download
  
  from ... import AutoTokenizer
  from ...utils import logging
  
  
  __lowerCamelCase  				=					"""block_records.npy"""
  
  
  __lowerCamelCase  				=					logging.get_logger(__name__)
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case							)   -> np.ndarray:
    import tensorflow.compat.va as tf
  
    _A           =    tf.data.TFRecordDataset(__snake_case						,				buffer_size=512 * 1_024 * 1_024							)
    _A           =    blocks_dataset.batch(__snake_case						,				drop_remainder=__snake_case							)
    _A           =    next(blocks_dataset.take(1							).as_numpy_iterator()							)
  
    return np_record
  
  
  
  class 		_snake_case      :
  
  
  
  
  
   """simple docstring"""
  
  
  
   def __init__(						self	,    a	,    a	,    a=2	,    a=1_0_0_0	,    a=1_0_0	,    a=1_0_0_0_0_0	,    )	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     from scann.scann_ops.py.scann_ops_pybind import builder as Builder
  
     _A           =    Builder(db=a	,    num_neighbors=a	,    distance_measure='''dot_product'''					)
     _A           =    builder.tree(
         num_leaves=a	,    num_leaves_to_search=a	,    training_sample_size=a					)
     _A           =    builder.score_ah(dimensions_per_block=a					)
  
     _A           =    builder.build()
  
  
  
   def lowercase_		(						self	,    a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A    ,   _A           =    self.searcher.search_batched(question_projection.detach().cpu()					)
     return retrieved_block_ids.astype('''int64'''					)
  
  
  
  
  
  
  
  class 		_snake_case      :
  
  
  
  
  
   """simple docstring"""
  
  
  
   def __init__(						self	,    a	,    a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     super().__init__()
     _A           =    block_records
     _A           =    tokenizer
   def __call__(						self	,    a	,    a	,    a	,    a=None	,    a="pt"					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    np.take(self.block_records	,    indices=a	,    axis=0					)
  
     _A           =    self.tokenizer.decode(question_input_ids[0]	,    skip_special_tokens=a					)
  
     _A           =    []
     _A           =    []
     for retrieved_block in retrieved_blocks:
       text.append(a					)
       text_pair.append(retrieved_block.decode()					)
  
     _A           =    self.tokenizer(
         a	,    a	,    padding=a	,    truncation=a	,    return_special_tokens_mask=a	,    max_length=a					)
     _A           =    concat_inputs.convert_to_tensors(a					)
  
     if answer_ids is not None:
       return self.block_has_answer(a	,    a					) + (concat_inputs_tensors,)
     else:
       return (None, None, None, concat_inputs_tensors)
   @classmethod
   def lowercase_		(						cls	,    a	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     if os.path.isdir(a					):
       _A           =    os.path.join(a	,    _REALM_BLOCK_RECORDS_FILENAME					)
     else:
       _A           =    hf_hub_download(
           repo_id=a	,    filename=_REALM_BLOCK_RECORDS_FILENAME	,    **a					)
     _A           =    np.load(a	,    allow_pickle=a					)
  
     _A           =    AutoTokenizer.from_pretrained(a	,    *a	,    **a					)
  
     return cls(a	,    a					)
   def lowercase_		(						self	,    a					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     np.save(os.path.join(a	,    _REALM_BLOCK_RECORDS_FILENAME					)	,    self.block_records					)
     # save tokenizer
     self.tokenizer.save_pretrained(a					)
  
  
  
   def lowercase_		(						self	,    a	,    a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    []
     _A           =    []
     _A           =    []
     _A           =    0
  
     for input_id in concat_inputs.input_ids:
       _A           =    input_id.tolist()
       # Check answers between two [SEP] tokens
       _A           =    input_id_list.index(self.tokenizer.sep_token_id					)
       _A           =    first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id					)
  
       start_pos.append([]					)
       end_pos.append([]					)
       for answer in answer_ids:
         for idx in range(first_sep_idx + 1	,    a					):
           if answer[0] == input_id_list[idx]:
             if input_id_list[idx : idx + len(a					)] == answer:
               start_pos[-1].append(a					)
               end_pos[-1].append(idx + len(a					) - 1					)
  
       if len(start_pos[-1]					) == 0:
         has_answers.append(a					)
       else:
         has_answers.append(a					)
         if len(start_pos[-1]					) > max_answers:
           _A           =    len(start_pos[-1]					)
  
          # Pad -1 to max_answers
     for start_pos_, end_pos_ in zip(a	,    a					):
       if len(start_pos_					) < max_answers:
         _A           =    [-1] * (max_answers - len(start_pos_					))
         start_pos_ += padded
         end_pos_ += padded
     return has_answers, start_pos, end_pos
style_context: 
  
  
  
  import copy
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  __lowerCamelCase  				=					logging.get_logger(__name__)
  
  __lowerCamelCase  				=					{}
  
  
  
  class 		_snake_case      (			lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			'''fsmt_decoder'''
   def __init__(						self	,    a=0	,    a=0					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     super().__init__()
     _A           =    vocab_size
     _A           =    bos_token_id
  
  
  
  
  
  
  
  class 		_snake_case      (			lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			'''fsmt'''
   lowerCamelCase_          =			{'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}
   def __init__(						self	,    a=["en", "de"]	,    a=4_2_0_2_4	,    a=4_2_0_2_4	,    a="relu"	,    a=1_0_2_4	,    a=2_0_0	,    a=1_0_2_4	,    a=4_0_9_6	,    a=1_2	,    a=1_6	,    a=0.0	,    a=4_0_9_6	,    a=1_2	,    a=1_6	,    a=0.0	,    a=0.0	,    a=0.1	,    a=0.0	,    a=0.02	,    a=2	,    a=True	,    a=True	,    a=False	,    a=5	,    a=1.0	,    a=False	,    a=True	,    a=1	,    a=0	,    a=2	,    a=2	,    **a	,    )	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    langs
     _A           =    src_vocab_size
     _A           =    tgt_vocab_size
     _A           =    d_model  # encoder_embed_dim and decoder_embed_dim
  
     _A           =    encoder_ffn_dim
     _A           =    _A           =    encoder_layers
     _A           =    encoder_attention_heads
     _A           =    encoder_layerdrop
     _A           =    decoder_layerdrop
     _A           =    decoder_ffn_dim
     _A           =    decoder_layers
     _A           =    decoder_attention_heads
     _A           =    max_position_embeddings
     _A           =    init_std  # Normal(0, this parameter)
     _A           =    activation_function
  
     _A           =    DecoderConfig(vocab_size=a	,    bos_token_id=a					)
     if "decoder" in common_kwargs:
       del common_kwargs["decoder"]
  
     _A           =    scale_embedding  # scale factor will be sqrt(d_model) if True
  
     # 3 Types of Dropout
     _A           =    attention_dropout
     _A           =    activation_dropout
     _A           =    dropout
  
     _A           =    use_cache
     super().__init__(
         pad_token_id=a	,    bos_token_id=a	,    eos_token_id=a	,    decoder_start_token_id=a	,    is_encoder_decoder=a	,    tie_word_embeddings=a	,    forced_eos_token_id=a	,    max_length=a	,    num_beams=a	,    length_penalty=a	,    early_stopping=a	,    **a	,    )
  
  
  
   def lowercase_		(						self					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    copy.deepcopy(self.__dict__					)
     _A           =    self.decoder.to_dict()
     _A           =    self.__class__.model_type
     return output