code: 
  '''simple docstring'''
  
  import os
  from typing import Optional, Union
  
  import numpy as np
  from huggingface_hub import hf_hub_download
  
  from ... import AutoTokenizer
  from ...utils import logging
  
  
  _UpperCAmelCase   :      List[Any] 						=  '''block_records.npy'''
  
  
  _UpperCAmelCase   :      Tuple 						=  logging.get_logger(__name__)
  
  
  
  
  
  
  
  def 						snake_case__	(      UpperCamelCase			,UpperCamelCase	) ->      np.ndarray:
  		import tensorflow.compat.va as tf
  
  		_UpperCamelCase       : Union[str, Any]    =      tf.data.TFRecordDataset(__lowerCAmelCase			,buffer_size=5_12 * 10_24 * 10_24	)
  		_UpperCamelCase       : List[str]    =      blocks_dataset.batch(__lowerCAmelCase			,drop_remainder=__lowerCAmelCase	)
  		_UpperCamelCase       : Any    =      next(blocks_dataset.take(1	).as_numpy_iterator()	)
  
  		return np_record
  
  class 			UpperCAmelCase				:
  
  
  
  
  
  		"""simple docstring"""
  
  
  
  
  
  		def __init__(    self  ,					_snake_case  ,					_snake_case  ,					_snake_case=2  ,					_snake_case=1000  ,					_snake_case=100  ,					_snake_case=100000  ,					)				->    Any:
  				from scann.scann_ops.py.scann_ops_pybind import builder as Builder
  
  				_UpperCamelCase       : Any    =      Builder(db=_snake_case  ,					num_neighbors=_snake_case  ,					distance_measure='''dot_product'''     )
  				_UpperCamelCase       : List[str]    =      builder.tree(
  				    num_leaves=_snake_case  ,					num_leaves_to_search=_snake_case  ,					training_sample_size=_snake_case     )
  				_UpperCamelCase       : Tuple    =      builder.score_ah(dimensions_per_block=_snake_case     )
  
  				_UpperCamelCase       : Union[str, Any]    =      builder.build()
  
  
  
  
  		def _lowercase  (    self  ,					_snake_case     )				->    Optional[Any]:
  				_UpperCamelCase       : Tuple    =      self.searcher.search_batched(question_projection.detach().cpu()     )
  				return retrieved_block_ids.astype('''int64'''     )
  
  
  
  
  
  
  
  class 			UpperCAmelCase				:
  
  
  
  
  
  		"""simple docstring"""
  
  
  
  
  
  		def __init__(    self  ,					_snake_case  ,					_snake_case     )				->    Tuple:
  				super().__init__()
  				_UpperCamelCase       : str    =      block_records
  				_UpperCamelCase       : List[str]    =      tokenizer
  		def __call__(    self  ,					_snake_case  ,					_snake_case  ,					_snake_case  ,					_snake_case=None  ,					_snake_case="pt"     )				->    Optional[int]:
  				_UpperCamelCase       : Union[str, Any]    =      np.take(self.block_records  ,					indices=_snake_case  ,					axis=0     )
  
  				_UpperCamelCase       : Dict    =      self.tokenizer.decode(question_input_ids[0]  ,					skip_special_tokens=_snake_case     )
  
  				_UpperCamelCase       : Dict    =      []
  				_UpperCamelCase       : Tuple    =      []
  				for retrieved_block in retrieved_blocks:
  						text.append(_snake_case     )
  						text_pair.append(retrieved_block.decode()     )
  
  				_UpperCamelCase       : List[str]    =      self.tokenizer(
  				    _snake_case  ,					_snake_case  ,					padding=_snake_case  ,					truncation=_snake_case  ,					return_special_tokens_mask=_snake_case  ,					max_length=_snake_case     )
  				_UpperCamelCase       : Dict    =      concat_inputs.convert_to_tensors(_snake_case     )
  
  				if answer_ids is not None:
  						return self.block_has_answer(_snake_case  ,					_snake_case     ) + (concat_inputs_tensors,)
  				else:
  						return (None, None, None, concat_inputs_tensors)
  		@classmethod
  		def _lowercase  (    cls  ,					_snake_case  ,					*_snake_case  ,					**_snake_case     )				->    int:
  
  
  
  
  
  
  				if os.path.isdir(_snake_case     ):
  						_UpperCamelCase       : Optional[Any]    =      os.path.join(_snake_case  ,					_REALM_BLOCK_RECORDS_FILENAME     )
  				else:
  						_UpperCamelCase       : List[str]    =      hf_hub_download(
  						    repo_id=_snake_case  ,					filename=_REALM_BLOCK_RECORDS_FILENAME  ,					**_snake_case     )
  				_UpperCamelCase       : Union[str, Any]    =      np.load(_snake_case  ,					allow_pickle=_snake_case     )
  
  				_UpperCamelCase       : List[Any]    =      AutoTokenizer.from_pretrained(_snake_case  ,					*_snake_case  ,					**_snake_case     )
  
  				return cls(_snake_case  ,					_snake_case     )
  		def _lowercase  (    self  ,					_snake_case     )				->    str:
  				np.save(os.path.join(_snake_case  ,					_REALM_BLOCK_RECORDS_FILENAME     )  ,					self.block_records     )
  				# save tokenizer
  				self.tokenizer.save_pretrained(_snake_case     )
  
  
  
  
  		def _lowercase  (    self  ,					_snake_case  ,					_snake_case     )				->    Union[str, Any]:
  				_UpperCamelCase       : Optional[int]    =      []
  				_UpperCamelCase       : Optional[int]    =      []
  				_UpperCamelCase       : Any    =      []
  				_UpperCamelCase       : int    =      0
  
  				for input_id in concat_inputs.input_ids:
  						_UpperCamelCase       : List[str]    =      input_id.tolist()
  						# Check answers between two [SEP] tokens
  						_UpperCamelCase       : Optional[int]    =      input_id_list.index(self.tokenizer.sep_token_id     )
  						_UpperCamelCase       : str    =      first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id     )
  
  						start_pos.append([]     )
  						end_pos.append([]     )
  						for answer in answer_ids:
  								for idx in range(first_sep_idx + 1  ,					_snake_case     ):
  										if answer[0] == input_id_list[idx]:
  												if input_id_list[idx : idx + len(_snake_case     )] == answer:
  														start_pos[-1].append(_snake_case     )
  														end_pos[-1].append(idx + len(_snake_case     ) - 1     )
  
  						if len(start_pos[-1]     ) == 0:
  								has_answers.append(_snake_case     )
  						else:
  								has_answers.append(_snake_case     )
  								if len(start_pos[-1]     ) > max_answers:
  										_UpperCamelCase       : str    =      len(start_pos[-1]     )
  
          # Pad -1 to max_answers
  				for start_pos_, end_pos_ in zip(_snake_case  ,					_snake_case     ):
  						if len(start_pos_     ) < max_answers:
  								_UpperCamelCase       : Dict    =      [-1] * (max_answers - len(start_pos_     ))
  								start_pos_ += padded
  								end_pos_ += padded
  				return has_answers, start_pos, end_pos
  
  
style_context: 
  '''simple docstring'''
  
  from collections import OrderedDict
  from typing import Mapping
  
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfig
  from ...utils import logging
  
  
  _UpperCAmelCase   :      List[Any] 						=  logging.get_logger(__name__)
  
  _UpperCAmelCase   :      List[Any] 						=  {
      """nghuyong/ernie-1.0-base-zh""": """https://huggingface.co/nghuyong/ernie-1.0-base-zh/resolve/main/config.json""",
      """nghuyong/ernie-2.0-base-en""": """https://huggingface.co/nghuyong/ernie-2.0-base-en/resolve/main/config.json""",
      """nghuyong/ernie-2.0-large-en""": """https://huggingface.co/nghuyong/ernie-2.0-large-en/resolve/main/config.json""",
      """nghuyong/ernie-3.0-base-zh""": """https://huggingface.co/nghuyong/ernie-3.0-base-zh/resolve/main/config.json""",
      """nghuyong/ernie-3.0-medium-zh""": """https://huggingface.co/nghuyong/ernie-3.0-medium-zh/resolve/main/config.json""",
      """nghuyong/ernie-3.0-mini-zh""": """https://huggingface.co/nghuyong/ernie-3.0-mini-zh/resolve/main/config.json""",
      """nghuyong/ernie-3.0-micro-zh""": """https://huggingface.co/nghuyong/ernie-3.0-micro-zh/resolve/main/config.json""",
      """nghuyong/ernie-3.0-nano-zh""": """https://huggingface.co/nghuyong/ernie-3.0-nano-zh/resolve/main/config.json""",
      """nghuyong/ernie-gram-zh""": """https://huggingface.co/nghuyong/ernie-gram-zh/resolve/main/config.json""",
      """nghuyong/ernie-health-zh""": """https://huggingface.co/nghuyong/ernie-health-zh/resolve/main/config.json""",
  }
  
  class 			UpperCAmelCase				(					a_ ):
  
  
  
  
  
  		"""simple docstring"""
  
  
  
  
  
  		A__  :   str          =    'ernie'
  		def __init__(    self  ,					_snake_case=30522  ,					_snake_case=768  ,					_snake_case=12  ,					_snake_case=12  ,					_snake_case=3072  ,					_snake_case="gelu"  ,					_snake_case=0.1  ,					_snake_case=0.1  ,					_snake_case=512  ,					_snake_case=2  ,					_snake_case=3  ,					_snake_case=False  ,					_snake_case=0.02  ,					_snake_case=1E-12  ,					_snake_case=0  ,					_snake_case="absolute"  ,					_snake_case=True  ,					_snake_case=None  ,					**_snake_case  ,					)				->    List[Any]:
  				super().__init__(pad_token_id=_snake_case  ,					**_snake_case     )
  
  				_UpperCamelCase       : Dict    =      vocab_size
  				_UpperCamelCase       : int    =      hidden_size
  				_UpperCamelCase       : List[Any]    =      num_hidden_layers
  				_UpperCamelCase       : Tuple    =      num_attention_heads
  				_UpperCamelCase       : List[str]    =      hidden_act
  				_UpperCamelCase       : Optional[int]    =      intermediate_size
  				_UpperCamelCase       : Tuple    =      hidden_dropout_prob
  				_UpperCamelCase       : Optional[int]    =      attention_probs_dropout_prob
  				_UpperCamelCase       : List[str]    =      max_position_embeddings
  				_UpperCamelCase       : Tuple    =      type_vocab_size
  				_UpperCamelCase       : Union[str, Any]    =      task_type_vocab_size
  				_UpperCamelCase       : Union[str, Any]    =      use_task_id
  				_UpperCamelCase       : Any    =      initializer_range
  				_UpperCamelCase       : Optional[int]    =      layer_norm_eps
  				_UpperCamelCase       : str    =      position_embedding_type
  				_UpperCamelCase       : Any    =      use_cache
  				_UpperCamelCase       : str    =      classifier_dropout
  
  
  
  
  
  
  
  class 			UpperCAmelCase				(					a_ ):
  
  
  
  
  
  		"""simple docstring"""
  
  
  
  
  
  		@property
  		def _lowercase  (    self     )				->    Mapping[str, Mapping[int, str]]:
  				if self.task == "multiple-choice":
  						_UpperCamelCase       : str    =      {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
  				else:
  						_UpperCamelCase       : Dict    =      {0: '''batch''', 1: '''sequence'''}
  				return OrderedDict(
  				    [
  				        ('''input_ids''', dynamic_axis),
  				        ('''attention_mask''', dynamic_axis),
  				        ('''token_type_ids''', dynamic_axis),
  				        ('''task_type_ids''', dynamic_axis),
  				    ]     )
  
  
  
  
  
  