






import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


snake_case_   :	Optional[Any]							      =logging.getLogger(__name__)





class        a__      :






		def __init__(      self						)				->				List[str]:
				__A    =							False






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__						)				->				Any:
				if not self.initialized:
						__A    =							RagRetriever(
						    lowercase__							, question_encoder_tokenizer=lowercase__							, generator_tokenizer=lowercase__							, index=lowercase__							, init_retrieval=lowercase__							, )
						__A    =							True






		def 					_lowerCamelCase					(      self						)				->				Tuple:
				self.retriever.index.init_index()






		def 					_lowerCamelCase					(      self						)				->				Tuple:
				# delete the old self.retriever object before assigning the new index
				del self.retriever
				__A    =							False






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__						)				->				Any:
				__A ,   __A    =							self.retriever._main_retrieve(lowercase__							, lowercase__						)
				__A    =							self.retriever.index.get_doc_dicts(lowercase__						)
				return doc_ids, retrieved_doc_embeds, doc_dicts





class        a__      (	lowerCAmelCase__		):






		def __init__(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__=None						)				->				Optional[int]:
				if index is not None and index.is_initialized() and len(lowercase__						) > 0:
						raise ValueError(
						    "When using Ray for distributed fine-tuning, "
						    "you'll need to provide the paths instead, "
						    "as the dataset and the index are loaded "
						    "separately. More info in examples/rag/use_own_knowledge_dataset.py "						)

				super().__init__(
				    lowercase__							, question_encoder_tokenizer=lowercase__							, generator_tokenizer=lowercase__							, index=lowercase__							, init_retrieval=lowercase__							, )

				__A    =							retrieval_workers
				__A    =							question_encoder_tokenizer
				__A    =							generator_tokenizer
				if len(self.retrieval_workers						) > 0:
						ray.get(
						    [
						        worker.create_rag_retriever.remote(lowercase__							, lowercase__							, lowercase__							, lowercase__						)
						        for worker in self.retrieval_workers
						    ]						)






		def 					_lowerCamelCase					(      self						)				->				Optional[int]:
				logger.info("initializing retrieval"						)

				if len(self.retrieval_workers						) > 0:
						ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]						)
				else:
						# Non-distributed training. Load index into this same process.
						self.index.init_index()






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__						)				->				Optional[Any]:
				if len(self.retrieval_workers						) > 0:
						# Select a random retrieval actor.
						__A    =							self.retrieval_workers[random.randint(0							, len(self.retrieval_workers						) - 1						)]
						__A ,   __A ,   __A    =							ray.get(
						    random_worker.retrieve.remote(lowercase__							, lowercase__						)						)
				else:
						__A ,   __A    =							self._main_retrieve(lowercase__							, lowercase__						)
						__A    =							self.index.get_doc_dicts(lowercase__						)
				return retrieved_doc_embeds, doc_ids, doc_dicts






		@classmethod
		def 					_lowerCamelCase					(      cls							, lowercase__							, lowercase__=None							, **lowercase__						)				->				str:
				return super(lowercase__							, cls						).get_tokenizers(lowercase__							, lowercase__							, **lowercase__						)






		@classmethod
		def 					_lowerCamelCase					(      cls							, lowercase__							, lowercase__							, lowercase__=None							, **lowercase__						)				->				Any:
				__A    =							kwargs.pop("config"							, lowercase__						) or RagConfig.from_pretrained(lowercase__							, **lowercase__						)
				__A    =							RagTokenizer.from_pretrained(lowercase__							, config=lowercase__						)
				__A    =							rag_tokenizer.question_encoder
				__A    =							rag_tokenizer.generator

				if indexed_dataset is not None:
						__A    =							"custom"
						__A    =							CustomHFIndex(config.retrieval_vector_size							, lowercase__						)
				else:
						__A    =							cls._build_index(lowercase__						)

				return cls(
				    lowercase__							, question_encoder_tokenizer=lowercase__							, generator_tokenizer=lowercase__							, retrieval_workers=lowercase__							, index=lowercase__							, )






		def 					_lowerCamelCase					(      self						)				->				Dict:
				logger.info("re-loading the new dataset with embeddings"						)
				# access from the training loop

				ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]						)

				# build the index object again
				__A    =							self._build_index(self.config						)

				ray.get(
				    [
				        worker.create_rag_retriever.remote(
				            self.config							, self.question_encoder_tokenizer							, self.generator_tokenizer							, lowercase__						)
				        for worker in self.retrieval_workers
				    ]						)



'''simple docstring'''

from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


__UpperCAmelCase												=   logging.get_logger(__name__)

__UpperCAmelCase												=   {
    'gpt2': 'https://huggingface.co/gpt2/resolve/main/config.json',
    'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/config.json',
    'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/config.json',
    'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/config.json',
    'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/config.json',
}


class 		_a     (							SCREAMING_SNAKE_CASE  ):







   """simple docstring"""




   A		= 'gpt2'
   A		= ['past_key_values']
   A		= {
       'hidden_size': 'n_embd',
       'max_position_embeddings': 'n_positions',
       'num_attention_heads': 'n_head',
       'num_hidden_layers': 'n_layer',
   }






   def __init__(       self    ,__SCREAMING_SNAKE_CASE=50257    ,__SCREAMING_SNAKE_CASE=1024    ,__SCREAMING_SNAKE_CASE=768    ,__SCREAMING_SNAKE_CASE=12    ,__SCREAMING_SNAKE_CASE=12    ,__SCREAMING_SNAKE_CASE=None    ,__SCREAMING_SNAKE_CASE="gelu_new"    ,__SCREAMING_SNAKE_CASE=0.1    ,__SCREAMING_SNAKE_CASE=0.1    ,__SCREAMING_SNAKE_CASE=0.1    ,__SCREAMING_SNAKE_CASE=1e-5    ,__SCREAMING_SNAKE_CASE=0.02    ,__SCREAMING_SNAKE_CASE="cls_index"    ,__SCREAMING_SNAKE_CASE=True    ,__SCREAMING_SNAKE_CASE=None    ,__SCREAMING_SNAKE_CASE=True    ,__SCREAMING_SNAKE_CASE=0.1    ,__SCREAMING_SNAKE_CASE=True    ,__SCREAMING_SNAKE_CASE=True    ,__SCREAMING_SNAKE_CASE=50256    ,__SCREAMING_SNAKE_CASE=50256    ,__SCREAMING_SNAKE_CASE=False    ,__SCREAMING_SNAKE_CASE=False    ,**__SCREAMING_SNAKE_CASE    ,):
    SCREAMING_SNAKE_CASE  :  Optional[Any]								=		vocab_size
    SCREAMING_SNAKE_CASE  :  Tuple								=		n_positions
    SCREAMING_SNAKE_CASE  :  int								=		n_embd
    SCREAMING_SNAKE_CASE  :  Dict								=		n_layer
    SCREAMING_SNAKE_CASE  :  Dict								=		n_head
    SCREAMING_SNAKE_CASE  :  List[Any]								=		n_inner
    SCREAMING_SNAKE_CASE  :  Dict								=		activation_function
    SCREAMING_SNAKE_CASE  :  Union[str, Any]								=		resid_pdrop
    SCREAMING_SNAKE_CASE  :  str								=		embd_pdrop
    SCREAMING_SNAKE_CASE  :  int								=		attn_pdrop
    SCREAMING_SNAKE_CASE  :  Tuple								=		layer_norm_epsilon
    SCREAMING_SNAKE_CASE  :  Tuple								=		initializer_range
    SCREAMING_SNAKE_CASE  :  Any								=		summary_type
    SCREAMING_SNAKE_CASE  :  List[Any]								=		summary_use_proj
    SCREAMING_SNAKE_CASE  :  int								=		summary_activation
    SCREAMING_SNAKE_CASE  :  int								=		summary_first_dropout
    SCREAMING_SNAKE_CASE  :  List[Any]								=		summary_proj_to_labels
    SCREAMING_SNAKE_CASE  :  Union[str, Any]								=		scale_attn_weights
    SCREAMING_SNAKE_CASE  :  Any								=		use_cache
    SCREAMING_SNAKE_CASE  :  List[str]								=		scale_attn_by_inverse_layer_idx
    SCREAMING_SNAKE_CASE  :  Any								=		reorder_and_upcast_attn

    SCREAMING_SNAKE_CASE  :  Tuple								=		bos_token_id
    SCREAMING_SNAKE_CASE  :  Optional[int]								=		eos_token_id

    super().__init__(bos_token_id=__SCREAMING_SNAKE_CASE    ,eos_token_id=__SCREAMING_SNAKE_CASE    ,**__SCREAMING_SNAKE_CASE      )



class 		_a     (							SCREAMING_SNAKE_CASE  ):







   """simple docstring"""




   def __init__(       self    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE = "default"    ,__SCREAMING_SNAKE_CASE = None    ,__SCREAMING_SNAKE_CASE = False    ,):
    super().__init__(__SCREAMING_SNAKE_CASE    ,task=__SCREAMING_SNAKE_CASE    ,patching_specs=__SCREAMING_SNAKE_CASE    ,use_past=__SCREAMING_SNAKE_CASE      )
    if not getattr(self._config    ,'pad_token_id'    ,__SCREAMING_SNAKE_CASE      ):
     # TODO: how to do that better?
     SCREAMING_SNAKE_CASE  :  int								=		0






   @property
   def 					__a			(       self      ):
    SCREAMING_SNAKE_CASE  :  Dict								=		OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}      )
    if self.use_past:
     self.fill_with_past_key_values_(__SCREAMING_SNAKE_CASE    ,direction='inputs'      )
     SCREAMING_SNAKE_CASE  :  Dict								=		{0: 'batch', 1: 'past_sequence + sequence'}
    else:
     SCREAMING_SNAKE_CASE  :  Optional[Any]								=		{0: 'batch', 1: 'sequence'}

    return common_inputs






   @property
   def 					__a			(       self      ):
    return self._config.n_layer






   @property
   def 					__a			(       self      ):
    return self._config.n_head






   def 					__a			(       self    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE = -1    ,__SCREAMING_SNAKE_CASE = -1    ,__SCREAMING_SNAKE_CASE = False    ,__SCREAMING_SNAKE_CASE = None    ,):
    SCREAMING_SNAKE_CASE  :  List[str]								=		super(__SCREAMING_SNAKE_CASE    ,self      ).generate_dummy_inputs(
        __SCREAMING_SNAKE_CASE    ,batch_size=__SCREAMING_SNAKE_CASE    ,seq_length=__SCREAMING_SNAKE_CASE    ,is_pair=__SCREAMING_SNAKE_CASE    ,framework=__SCREAMING_SNAKE_CASE      )

    # We need to order the input in the way they appears in the forward()
    SCREAMING_SNAKE_CASE  :  int								=		OrderedDict({'input_ids': common_inputs['input_ids']}      )

    # Need to add the past_keys
    if self.use_past:
     if not is_torch_available():
      raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'      )
     else:
      import torch

      SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE  :  List[str]								=		common_inputs['input_ids'].shape
      # Not using the same length for past_key_values
      SCREAMING_SNAKE_CASE  :  str								=		seqlen + 2
      SCREAMING_SNAKE_CASE  :  Dict								=		(
          batch,
          self.num_attention_heads,
          past_key_values_length,
          self._config.hidden_size // self.num_attention_heads,
      )
      SCREAMING_SNAKE_CASE  :  Union[str, Any]								=		[
          (torch.zeros(__SCREAMING_SNAKE_CASE      ), torch.zeros(__SCREAMING_SNAKE_CASE      )) for _ in range(self.num_layers      )
      ]

    SCREAMING_SNAKE_CASE  :  Optional[int]								=		common_inputs['attention_mask']
    if self.use_past:
     SCREAMING_SNAKE_CASE  :  Tuple								=		ordered_inputs['attention_mask'].dtype
     SCREAMING_SNAKE_CASE  :  Any								=		torch.cat(
         [ordered_inputs['attention_mask'], torch.ones(__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE    ,dtype=__SCREAMING_SNAKE_CASE      )]    ,dim=1      )

    return ordered_inputs






   @property
   def 					__a			(       self      ):
    return 13


