






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__				   =		logging.get_logger(__name__)

UpperCAmelCase__				   =		{}






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      __snake_case       =  '''llama'''
      __snake_case       =  ['''past_key_values''']







      def __init__(      self      :						Optional[Any] ,					__UpperCAmelCase      :						Union[str, Any]=32_000 ,					__UpperCAmelCase      :						str=4_096 ,					__UpperCAmelCase      :						int=11_008 ,					__UpperCAmelCase      :						Tuple=32 ,					__UpperCAmelCase      :						Optional[int]=32 ,					__UpperCAmelCase      :						List[str]=None ,					__UpperCAmelCase      :						Union[str, Any]="silu" ,					__UpperCAmelCase      :						Tuple=2_048 ,					__UpperCAmelCase      :						Optional[Any]=0.02 ,					__UpperCAmelCase      :						Any=1e-6 ,					__UpperCAmelCase      :						Union[str, Any]=True ,					__UpperCAmelCase      :						Optional[int]=0 ,					__UpperCAmelCase      :						Optional[int]=1 ,					__UpperCAmelCase      :						Optional[int]=2 ,					__UpperCAmelCase      :						Tuple=1 ,					__UpperCAmelCase      :						List[str]=False ,					__UpperCAmelCase      :						Tuple=None ,					**__UpperCAmelCase      :						Tuple ,					)    ->str:




       """simple docstring"""


       a		  =      vocab_size
       a		  =      max_position_embeddings
       a		  =      hidden_size
       a		  =      intermediate_size
       a		  =      num_hidden_layers
       a		  =      num_attention_heads

       # for backward compatibility
       if num_key_value_heads is None:
        a		  =      num_attention_heads

       a		  =      num_key_value_heads
       a		  =      hidden_act
       a		  =      initializer_range
       a		  =      rms_norm_eps
       a		  =      pretraining_tp
       a		  =      use_cache
       a		  =      rope_scaling
       self._rope_scaling_validation()

       super().__init__(
           pad_token_id=__UpperCAmelCase ,					bos_token_id=__UpperCAmelCase ,					eos_token_id=__UpperCAmelCase ,					tie_word_embeddings=__UpperCAmelCase ,					**__UpperCAmelCase ,					)







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Tuple:




       """simple docstring"""


       if self.rope_scaling is None:
        return

       if not isinstance(self.rope_scaling ,					__UpperCAmelCase  ) or len(self.rope_scaling  ) != 2:
        raise ValueError(
            '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
            F"""got {self.rope_scaling}"""  )
       a		  =      self.rope_scaling.get('''type''' ,					__UpperCAmelCase  )
       a		  =      self.rope_scaling.get('''factor''' ,					__UpperCAmelCase  )
       if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
        raise ValueError(
            F"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""  )
       if rope_scaling_factor is None or not isinstance(__UpperCAmelCase ,					__UpperCAmelCase  ) or rope_scaling_factor <= 1.0:
        raise ValueError(F"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""  )







'''simple docstring'''





import argparse
from pathlib import Path

from transformers import AutoConfig, AutoTokenizer, RagConfig, RagSequenceForGeneration, RagTokenForGeneration

def        a_     (							_lowerCAmelCase       ,_lowerCAmelCase       ,_lowerCAmelCase       ,_lowerCAmelCase       ,_lowerCAmelCase = None       ,_lowerCAmelCase = None       ,_lowerCAmelCase = None       ,)      ->      Optional[int]:
  if config_name_or_path is None:
    __lowerCamelCase       :       List[Any]           = 'facebook/rag-token-base' if model_type == 'rag_token' else 'facebook/rag-sequence-base'

  if generator_tokenizer_name_or_path is None:
    __lowerCamelCase       :       Optional[int]           = generator_name_or_path

  if question_encoder_tokenizer_name_or_path is None:
    __lowerCamelCase       :       Tuple           = question_encoder_name_or_path

  __lowerCamelCase       :       Union[str, Any]           = RagTokenForGeneration if model_type == 'rag_token' else RagSequenceForGeneration

  # Save model.
  __lowerCamelCase       :       Tuple           = RagConfig.from_pretrained(_lowerCAmelCase       )
  __lowerCamelCase       :       List[Any]           = AutoConfig.from_pretrained(_lowerCAmelCase       )
  __lowerCamelCase       :       Union[str, Any]           = AutoConfig.from_pretrained(_lowerCAmelCase       )

  __lowerCamelCase       :       Tuple           = gen_config
  __lowerCamelCase       :       List[Any]           = question_encoder_config

  __lowerCamelCase       :       str           = model_class.from_pretrained_question_encoder_generator(
      _lowerCAmelCase       ,_lowerCAmelCase       ,config=_lowerCAmelCase       )
  rag_model.save_pretrained(_lowerCAmelCase       )

  # Sanity check.
  model_class.from_pretrained(_lowerCAmelCase       )

  # Save tokenizers.
  __lowerCamelCase       :       List[str]           = AutoTokenizer.from_pretrained(_lowerCAmelCase       )
  gen_tokenizer.save_pretrained(dest_dir / 'generator_tokenizer/'       )
  __lowerCamelCase       :       Optional[int]           = AutoTokenizer.from_pretrained(_lowerCAmelCase       )
  question_encoder_tokenizer.save_pretrained(dest_dir / 'question_encoder_tokenizer/'       )


if __name__ == "__main__":
   _UpperCamelCase											=						argparse.ArgumentParser()
   parser.add_argument(
       '--model_type',
       choices=['rag_sequence', 'rag_token'],
       required=True,
       type=str,
       help='RAG model type: rag_sequence, rag_token',
   )
   parser.add_argument('--dest', type=str, required=True, help='Path to the output checkpoint directory.')
   parser.add_argument('--generator_name_or_path', type=str, required=True, help='Generator model identifier')
   parser.add_argument(
       '--question_encoder_name_or_path', type=str, required=True, help='Question encoder model identifier'
   )

   parser.add_argument(
       '--generator_tokenizer_name_or_path',
       type=str,
       help='Generator tokenizer identifier, if not specified, resolves to ``generator_name_or_path``',
   )
   parser.add_argument(
       '--question_encoder_tokenizer_name_or_path',
       type=str,
       help='Question encoder tokenizer identifier, if not specified, resolves to ``question_encoder_name_or_path``',
   )
   parser.add_argument(
       '--config_name_or_path',
       type=str,
       help=(
           'Identifier of the model config to use, if not provided, resolves to a base config for a given'
           ' ``model_type``'
       ),
   )

   _UpperCamelCase											=						parser.parse_args()

   _UpperCamelCase											=						Path(args.dest)
   dest_dir.mkdir(exist_ok=True)

   consolidate(
       args.model_type,
       args.generator_name_or_path,
       args.question_encoder_name_or_path,
       dest_dir,
       args.config_name_or_path,
       args.generator_tokenizer_name_or_path,
       args.question_encoder_tokenizer_name_or_path,
   )






