





import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


a__:      Any									=		logging.getLogger(__name__)




class     SCREAMING_SNAKE_CASE__ :



 def __init__(							self ):
        A__											=						False



 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase ):
        if not self.initialized:
               A__											=						RagRetriever(
                   __lowerCamelCase,question_encoder_tokenizer=__lowerCamelCase,generator_tokenizer=__lowerCamelCase,index=__lowerCamelCase,init_retrieval=__lowerCamelCase,)
               A__											=						True



 def 				UpperCamelCase			(							self ):
        self.retriever.index.init_index()




 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase ):
        A__					,	A__											=						self.retriever._main_retrieve(__lowerCamelCase,__lowerCamelCase )
        return doc_ids, retrieved_doc_embeds




class     SCREAMING_SNAKE_CASE__ (						UpperCamelCase__      ):



 def __init__(							self,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase=None ):
        if index is not None and index.is_initialized() and len(__lowerCamelCase ) > 0:
               raise ValueError(
                   '''When using Ray for distributed fine-tuning, '''
                   '''you\'ll need to provide the paths instead, '''
                   '''as the dataset and the index are loaded '''
                   '''separately. More info in examples/rag/use_own_knowledge_dataset.py ''' )
        super().__init__(
            __lowerCamelCase,question_encoder_tokenizer=__lowerCamelCase,generator_tokenizer=__lowerCamelCase,index=__lowerCamelCase,init_retrieval=__lowerCamelCase,)
        A__											=						retrieval_workers
        if len(self.retrieval_workers ) > 0:
               ray.get(
                   [
                       worker.create_rag_retriever.remote(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase )
                       for worker in self.retrieval_workers
                   ] )



 def 				UpperCamelCase			(							self ):
        logger.info('''initializing retrieval''' )

        if len(self.retrieval_workers ) > 0:
               ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers] )
        else:
               # Non-distributed training. Load index into this same process.
               self.index.init_index()



 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase ):
        if len(self.retrieval_workers ) > 0:
               # Select a random retrieval actor.
               A__											=						self.retrieval_workers[random.randint(0,len(self.retrieval_workers ) - 1 )]
               A__					,	A__											=						ray.get(random_worker.retrieve.remote(__lowerCamelCase,__lowerCamelCase ) )
        else:
               A__					,	A__											=						self._main_retrieve(__lowerCamelCase,__lowerCamelCase )
        return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(__lowerCamelCase )



 @classmethod
 def 				UpperCamelCase			(							cls,__lowerCamelCase,__lowerCamelCase=None,**__lowerCamelCase ):
        return super(__lowerCamelCase,cls ).get_tokenizers(__lowerCamelCase,__lowerCamelCase,**__lowerCamelCase )




 @classmethod
 def 				UpperCamelCase			(							cls,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase=None,**__lowerCamelCase ):
        A__											=						kwargs.pop('''config''',__lowerCamelCase ) or RagConfig.from_pretrained(__lowerCamelCase,**__lowerCamelCase )
        A__											=						RagTokenizer.from_pretrained(__lowerCamelCase,config=__lowerCamelCase )
        A__											=						rag_tokenizer.question_encoder
        A__											=						rag_tokenizer.generator
        if indexed_dataset is not None:
               A__											=						'''custom'''
               A__											=						CustomHFIndex(config.retrieval_vector_size,__lowerCamelCase )
        else:
               A__											=						cls._build_index(__lowerCamelCase )
        return cls(
            __lowerCamelCase,question_encoder_tokenizer=__lowerCamelCase,generator_tokenizer=__lowerCamelCase,retrieval_workers=__lowerCamelCase,index=__lowerCamelCase,)












import unittest

from transformers import AutoConfig, AutoTokenizer, BertConfig, TensorType, is_flax_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, slow


if is_flax_available():
       import jax

       from transformers.models.auto.modeling_flax_auto import FlaxAutoModel
       from transformers.models.bert.modeling_flax_bert import FlaxBertModel
       from transformers.models.roberta.modeling_flax_roberta import FlaxRobertaModel




@require_flax
class     SCREAMING_SNAKE_CASE__ (						unittest.TestCase      ):



 @slow
 def 				UpperCamelCase			(							self ):
        for model_name in ["bert-base-cased", "bert-large-uncased"]:
               with self.subTest(__lowerCamelCase ):
                      A__											=						AutoConfig.from_pretrained(__lowerCamelCase )
                      self.assertIsNotNone(__lowerCamelCase )
                      self.assertIsInstance(__lowerCamelCase,__lowerCamelCase )

                      A__											=						FlaxAutoModel.from_pretrained(__lowerCamelCase )
                      self.assertIsNotNone(__lowerCamelCase )
                      self.assertIsInstance(__lowerCamelCase,__lowerCamelCase )



 @slow
 def 				UpperCamelCase			(							self ):
        for model_name in ["roberta-base", "roberta-large"]:
               with self.subTest(__lowerCamelCase ):
                      A__											=						AutoConfig.from_pretrained(__lowerCamelCase )
                      self.assertIsNotNone(__lowerCamelCase )
                      self.assertIsInstance(__lowerCamelCase,__lowerCamelCase )

                      A__											=						FlaxAutoModel.from_pretrained(__lowerCamelCase )
                      self.assertIsNotNone(__lowerCamelCase )
                      self.assertIsInstance(__lowerCamelCase,__lowerCamelCase )



 @slow
 def 				UpperCamelCase			(							self ):
        for model_name in ["bert-base-cased", "bert-large-uncased"]:
               A__											=						AutoTokenizer.from_pretrained(__lowerCamelCase )
               A__											=						FlaxBertModel.from_pretrained(__lowerCamelCase )
               A__											=						tokenizer('''Do you support jax jitted function?''',return_tensors=TensorType.JAX )

               @jax.jit
               def eval(**__lowerCamelCase ):
                      return model(**__lowerCamelCase )

               eval(**__lowerCamelCase ).block_until_ready()



 @slow
 def 				UpperCamelCase			(							self ):
        for model_name in ["roberta-base", "roberta-large"]:
               A__											=						AutoTokenizer.from_pretrained(__lowerCamelCase )
               A__											=						FlaxRobertaModel.from_pretrained(__lowerCamelCase )
               A__											=						tokenizer('''Do you support jax jitted function?''',return_tensors=TensorType.JAX )

               @jax.jit
               def eval(**__lowerCamelCase ):
                      return model(**__lowerCamelCase )

               eval(**__lowerCamelCase ).block_until_ready()



 def 				UpperCamelCase			(							self ):
        with self.assertRaisesRegex(
            __lowerCamelCase,'''bert-base is not a local folder and is not a valid model identifier''' ):
               A__											=						FlaxAutoModel.from_pretrained('''bert-base''' )



 def 				UpperCamelCase			(							self ):
        with self.assertRaisesRegex(
            __lowerCamelCase,r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
               A__											=						FlaxAutoModel.from_pretrained(__lowerCamelCase,revision='''aaaaaa''' )



 def 				UpperCamelCase			(							self ):
        with self.assertRaisesRegex(
            __lowerCamelCase,'''hf-internal-testing/config-no-model does not appear to have a file named flax_model.msgpack''',):
               A__											=						FlaxAutoModel.from_pretrained('''hf-internal-testing/config-no-model''' )




 def 				UpperCamelCase			(							self ):
        with self.assertRaisesRegex(__lowerCamelCase,'''Use `from_pt=True` to load this model''' ):
               A__											=						FlaxAutoModel.from_pretrained('''hf-internal-testing/tiny-bert-pt-only''' )





