


# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path

from .config_args import default_config_file, load_config_from_file
from .config_utils import SubcommandHelpFormatter


__lowerCamelCase   	=  '''Update an existing config file with the latest defaults while maintaining the old configuration.'''



def 			_a      (    __UpperCamelCase			):
     a_     :  Optional[int]				      =     args.config_file
     if config_file is None and Path(__UpperCamelCase			).exists():
          a_     :  Optional[int]				      =     default_config_file
     elif not Path(__UpperCamelCase			).exists():
          raise ValueError(F'''The passed config file located at {config_file} doesn\'t exist.'''			)
     a_     :  List[str]				      =     load_config_from_file(__UpperCamelCase			)

     if config_file.endswith(""".json"""			):
          config.to_json_file(__UpperCamelCase			)
     else:
          config.to_yaml_file(__UpperCamelCase			)
     return config_file



def 			_a      (    __UpperCamelCase	,	__UpperCamelCase			):
     a_     :  List[Any]				      =     parser.add_parser("""update"""	,	parents=__UpperCamelCase	,	help=__UpperCamelCase	,	formatter_class=__UpperCamelCase			)
     parser.add_argument(
         """--config_file"""	,	default=__UpperCamelCase	,	help=(
             """The path to the config file to update. Will default to a file named default_config.yaml in the cache """
             """location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have """
             """such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed """
             """with 'huggingface'."""
         )	,	)

     parser.set_defaults(func=__UpperCamelCase			)
     return parser



def 			_a      (    __UpperCamelCase			):
     a_     :  str				      =     update_config(__UpperCamelCase			)
     print(F'''Sucessfully updated the configuration file at {config_file}.'''			)




'''simple docstring'''




import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


__snake_case  : Optional[int]		       =    logging.getLogger(__name__)




class       __UpperCAmelCase							:





       '''simple docstring'''



       def __init__( self					)						->	List[str]:
           A_      =  False

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)						->	str:
           if not self.initialized:
               A_      =  RagRetriever(
                   _SCREAMING_SNAKE_CASE       ,	question_encoder_tokenizer=_SCREAMING_SNAKE_CASE       ,	generator_tokenizer=_SCREAMING_SNAKE_CASE       ,	index=_SCREAMING_SNAKE_CASE       ,	init_retrieval=_SCREAMING_SNAKE_CASE       ,	)
               A_      =  True

       def    __A			( self					)						->	List[Any]:
           self.retriever.index.init_index()

       def    __A			( self					)						->	Dict:
           # delete the old self.retriever object before assigning the new index
           del self.retriever
           A_      =  False

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)						->	Optional[int]:
           A_  ,A_      =  self.retriever._main_retrieve(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
           A_      =  self.retriever.index.get_doc_dicts(_SCREAMING_SNAKE_CASE					)
           return doc_ids, retrieved_doc_embeds, doc_dicts




class       __UpperCAmelCase							(							_UpperCamelCase	):





       '''simple docstring'''



       def __init__( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None					)						->	List[Any]:
           if index is not None and index.is_initialized() and len(_SCREAMING_SNAKE_CASE					) > 0:
               raise ValueError(
                   '''When using Ray for distributed fine-tuning, '''
                   '''you\'ll need to provide the paths instead, '''
                   '''as the dataset and the index are loaded '''
                   '''separately. More info in examples/rag/use_own_knowledge_dataset.py '''					)

           super().__init__(
               _SCREAMING_SNAKE_CASE       ,	question_encoder_tokenizer=_SCREAMING_SNAKE_CASE       ,	generator_tokenizer=_SCREAMING_SNAKE_CASE       ,	index=_SCREAMING_SNAKE_CASE       ,	init_retrieval=_SCREAMING_SNAKE_CASE       ,	)

           A_      =  retrieval_workers
           A_      =  question_encoder_tokenizer
           A_      =  generator_tokenizer
           if len(self.retrieval_workers					) > 0:
               ray.get(
                   [
                       worker.create_rag_retriever.remote(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
                       for worker in self.retrieval_workers
                   ]					)

       def    __A			( self					)						->	List[Any]:
           logger.info('''initializing retrieval'''					)

           if len(self.retrieval_workers					) > 0:
               ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]					)
           else:
               # Non-distributed training. Load index into this same process.
               self.index.init_index()

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)						->	Optional[int]:
           if len(self.retrieval_workers					) > 0:
               # Select a random retrieval actor.
               A_      =  self.retrieval_workers[random.randint(0       ,	len(self.retrieval_workers					) - 1					)]
               A_  ,A_  ,A_      =  ray.get(
                   random_worker.retrieve.remote(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)					)
           else:
               A_  ,A_      =  self._main_retrieve(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
               A_      =  self.index.get_doc_dicts(_SCREAMING_SNAKE_CASE					)
           return retrieved_doc_embeds, doc_ids, doc_dicts

       @classmethod
       def    __A			( cls       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None       ,	**_SCREAMING_SNAKE_CASE					)						->	List[Any]:
           return super(_SCREAMING_SNAKE_CASE       ,	cls					).get_tokenizers(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)

       @classmethod
       def    __A			( cls       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None       ,	**_SCREAMING_SNAKE_CASE					)						->	int:
           A_      =  kwargs.pop('''config'''       ,	_SCREAMING_SNAKE_CASE					) or RagConfig.from_pretrained(_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)
           A_      =  RagTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE       ,	config=_SCREAMING_SNAKE_CASE					)
           A_      =  rag_tokenizer.question_encoder
           A_      =  rag_tokenizer.generator

           if indexed_dataset is not None:
               A_      =  '''custom'''
               A_      =  CustomHFIndex(config.retrieval_vector_size       ,	_SCREAMING_SNAKE_CASE					)
           else:
               A_      =  cls._build_index(_SCREAMING_SNAKE_CASE					)

           return cls(
               _SCREAMING_SNAKE_CASE       ,	question_encoder_tokenizer=_SCREAMING_SNAKE_CASE       ,	generator_tokenizer=_SCREAMING_SNAKE_CASE       ,	retrieval_workers=_SCREAMING_SNAKE_CASE       ,	index=_SCREAMING_SNAKE_CASE       ,	)

       def    __A			( self					)						->	Optional[int]:
           logger.info('''re-loading the new dataset with embeddings'''					)
           # access from the training loop

           ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]					)

           # build the index object again
           A_      =  self._build_index(self.config					)

           ray.get(
               [
                   worker.create_rag_retriever.remote(
                       self.config       ,	self.question_encoder_tokenizer       ,	self.generator_tokenizer       ,	_SCREAMING_SNAKE_CASE					)
                   for worker in self.retrieval_workers
               ]					)






