



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
     import torch

if is_vision_available():
     from PIL import Image

     from transformers import MobileNetVaImageProcessor



class 		_lowercase		(							unittest.TestCase						):

   def __init__(   self      :							Dict				, __lowerCAmelCase      :							List[str]				, __lowerCAmelCase      :							Optional[Any]=7				, __lowerCAmelCase      :							int=3				, __lowerCAmelCase      :							Optional[int]=18				, __lowerCAmelCase      :							List[str]=30				, __lowerCAmelCase      :							List[Any]=400				, __lowerCAmelCase      :							Any=True				, __lowerCAmelCase      :							str=None				, __lowerCAmelCase      :							Union[str, Any]=True				, __lowerCAmelCase      :							Tuple=None				, )							->					Any:


    """simple docstring"""






    a							=		size if size is not None else {"shortest_edge": 20}
    a							=		crop_size if crop_size is not None else {"height": 18, "width": 18}
    a							=		parent
    a							=		batch_size
    a							=		num_channels
    a							=		image_size
    a							=		min_resolution
    a							=		max_resolution
    a							=		do_resize
    a							=		size
    a							=		do_center_crop
    a							=		crop_size




   def 		A				(   self      :							Optional[int]				)							->					Any:


    """simple docstring"""






    return {
        "do_resize": self.do_resize,
        "size": self.size,
        "do_center_crop": self.do_center_crop,
        "crop_size": self.crop_size,
    }



@require_torch
@require_vision
class 		_lowercase		(							UpperCAmelCase__, unittest.TestCase						):
   _UpperCAmelCase  						=      MobileNetVaImageProcessor if is_vision_available() else None

   def 		A				(   self      :							Any				)							->					Optional[int]:


    """simple docstring"""






    a							=		MobileNetVaImageProcessingTester(self				)

   @property
   def 		A				(   self      :							List[str]				)							->					List[str]:


    """simple docstring"""






    return self.image_processor_tester.prepare_image_processor_dict()

   def 		A				(   self      :							Optional[int]				)							->					Union[str, Any]:


    """simple docstring"""






    a							=		self.image_processing_class(**self.image_processor_dict				)
    self.assertTrue(hasattr(__lowerCAmelCase				, "do_resize"				)				)
    self.assertTrue(hasattr(__lowerCAmelCase				, "size"				)				)
    self.assertTrue(hasattr(__lowerCAmelCase				, "do_center_crop"				)				)
    self.assertTrue(hasattr(__lowerCAmelCase				, "center_crop"				)				)

   def 		A				(   self      :							List[str]				)							->					Dict:


    """simple docstring"""






    a							=		self.image_processing_class.from_dict(self.image_processor_dict				)
    self.assertEqual(image_processor.size				, {"shortest_edge": 20}				)
    self.assertEqual(image_processor.crop_size				, {"height": 18, "width": 18}				)

    a							=		self.image_processing_class.from_dict(self.image_processor_dict				, size=42				, crop_size=84				)
    self.assertEqual(image_processor.size				, {"shortest_edge": 42}				)
    self.assertEqual(image_processor.crop_size				, {"height": 84, "width": 84}				)

   def 		A				(   self      :							Dict				)							->					Tuple:


    """simple docstring"""






    pass

   def 		A				(   self      :							int				)							->					Optional[int]:


    """simple docstring"""






    a							=		self.image_processing_class(**self.image_processor_dict				)
    # create random PIL images
    a							=		prepare_image_inputs(self.image_processor_tester				, equal_resolution=__lowerCAmelCase				)
    for image in image_inputs:
     self.assertIsInstance(__lowerCAmelCase				, Image.Image				)

    # Test not batched input
    a							=		image_processing(image_inputs[0]				, return_tensors="pt"				).pixel_values
    self.assertEqual(
        encoded_images.shape				, (
            1,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size["height"],
            self.image_processor_tester.crop_size["width"],
        )				, )

    # Test batched
    a							=		image_processing(__lowerCAmelCase				, return_tensors="pt"				).pixel_values
    self.assertEqual(
        encoded_images.shape				, (
            self.image_processor_tester.batch_size,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size["height"],
            self.image_processor_tester.crop_size["width"],
        )				, )

   def 		A				(   self      :							List[Any]				)							->					Dict:


    """simple docstring"""






    a							=		self.image_processing_class(**self.image_processor_dict				)
    # create random numpy tensors
    a							=		prepare_image_inputs(self.image_processor_tester				, equal_resolution=__lowerCAmelCase				, numpify=__lowerCAmelCase				)
    for image in image_inputs:
     self.assertIsInstance(__lowerCAmelCase				, np.ndarray				)

    # Test not batched input
    a							=		image_processing(image_inputs[0]				, return_tensors="pt"				).pixel_values
    self.assertEqual(
        encoded_images.shape				, (
            1,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size["height"],
            self.image_processor_tester.crop_size["width"],
        )				, )

    # Test batched
    a							=		image_processing(__lowerCAmelCase				, return_tensors="pt"				).pixel_values
    self.assertEqual(
        encoded_images.shape				, (
            self.image_processor_tester.batch_size,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size["height"],
            self.image_processor_tester.crop_size["width"],
        )				, )




   def 		A				(   self      :							Tuple				)							->					Tuple:


    """simple docstring"""






    a							=		self.image_processing_class(**self.image_processor_dict				)
    # create random PyTorch tensors
    a							=		prepare_image_inputs(self.image_processor_tester				, equal_resolution=__lowerCAmelCase				, torchify=__lowerCAmelCase				)
    for image in image_inputs:
     self.assertIsInstance(__lowerCAmelCase				, torch.Tensor				)

    # Test not batched input
    a							=		image_processing(image_inputs[0]				, return_tensors="pt"				).pixel_values
    self.assertEqual(
        encoded_images.shape				, (
            1,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size["height"],
            self.image_processor_tester.crop_size["width"],
        )				, )

    # Test batched
    a							=		image_processing(__lowerCAmelCase				, return_tensors="pt"				).pixel_values
    self.assertEqual(
        encoded_images.shape				, (
            self.image_processor_tester.batch_size,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size["height"],
            self.image_processor_tester.crop_size["width"],
        )				, )







import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


UpperCamelCase__       				=	logging.getLogger(__name__)
class 							__lowercase   :



     def __init__(  self   :							List[str]       ):
       a_		    =  False



     def 				__magic_name__		(  self   :							Union[str, Any]      ,		lowercase__   :							List[str]      ,		lowercase__   :							Optional[int]      ,		lowercase__   :							Optional[Any]      ,		lowercase__   :							Union[str, Any]       ):
       if not self.initialized:
         a_		    =  RagRetriever(
             lowercase__      ,		question_encoder_tokenizer=lowercase__      ,		generator_tokenizer=lowercase__      ,		index=lowercase__      ,		init_retrieval=lowercase__      ,		)
         a_		    =  True



     def 				__magic_name__		(  self   :							List[Any]       ):
       self.retriever.index.init_index()



     def 				__magic_name__		(  self   :							List[Any]       ):
       # delete the old self.retriever object before assigning the new index
       del self.retriever
       a_		    =  False



     def 				__magic_name__		(  self   :							Optional[Any]      ,		lowercase__   :							List[str]      ,		lowercase__   :							Dict       ):
       a_    , a_		    =  self.retriever._main_retrieve(lowercase__      ,		lowercase__       )
       a_		    =  self.retriever.index.get_doc_dicts(lowercase__       )
       return doc_ids, retrieved_doc_embeds, doc_dicts



class 							__lowercase   (			a__    ):



     def __init__(  self   :							List[Any]      ,		lowercase__   :							Optional[int]      ,		lowercase__   :							List[str]      ,		lowercase__   :							Tuple      ,		lowercase__   :							Dict      ,		lowercase__   :							Dict=None       ):
       if index is not None and index.is_initialized() and len(lowercase__       ) > 0:
         raise ValueError(
             '''When using Ray for distributed fine-tuning, '''
             '''you\'ll need to provide the paths instead, '''
             '''as the dataset and the index are loaded '''
             '''separately. More info in examples/rag/use_own_knowledge_dataset.py '''       )

       super().__init__(
           lowercase__      ,		question_encoder_tokenizer=lowercase__      ,		generator_tokenizer=lowercase__      ,		index=lowercase__      ,		init_retrieval=lowercase__      ,		)

       a_		    =  retrieval_workers
       a_		    =  question_encoder_tokenizer
       a_		    =  generator_tokenizer
       if len(self.retrieval_workers       ) > 0:
         ray.get(
             [
                 worker.create_rag_retriever.remote(lowercase__      ,		lowercase__      ,		lowercase__      ,		lowercase__       )
                 for worker in self.retrieval_workers
             ]       )



     def 				__magic_name__		(  self   :							int       ):
       logger.info('''initializing retrieval'''       )

       if len(self.retrieval_workers       ) > 0:
         ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]       )
       else:
         # Non-distributed training. Load index into this same process.
         self.index.init_index()



     def 				__magic_name__		(  self   :							str      ,		lowercase__   :							List[str]      ,		lowercase__   :							Any       ):
       if len(self.retrieval_workers       ) > 0:
         # Select a random retrieval actor.
         a_		    =  self.retrieval_workers[random.randint(0      ,		len(self.retrieval_workers       ) - 1       )]
         a_    , a_    , a_		    =  ray.get(
             random_worker.retrieve.remote(lowercase__      ,		lowercase__       )       )
       else:
         a_    , a_		    =  self._main_retrieve(lowercase__      ,		lowercase__       )
         a_		    =  self.index.get_doc_dicts(lowercase__       )
       return retrieved_doc_embeds, doc_ids, doc_dicts



     @classmethod
     def 				__magic_name__		(  cls   :							Dict      ,		lowercase__   :							Optional[Any]      ,		lowercase__   :							List[str]=None      ,		**lowercase__   :							int       ):
       return super(lowercase__      ,		cls       ).get_tokenizers(lowercase__      ,		lowercase__      ,		**lowercase__       )



     @classmethod
     def 				__magic_name__		(  cls   :							Tuple      ,		lowercase__   :							Any      ,		lowercase__   :							str      ,		lowercase__   :							List[Any]=None      ,		**lowercase__   :							Union[str, Any]       ):
       a_		    =  kwargs.pop('''config'''      ,		lowercase__       ) or RagConfig.from_pretrained(lowercase__      ,		**lowercase__       )
       a_		    =  RagTokenizer.from_pretrained(lowercase__      ,		config=lowercase__       )
       a_		    =  rag_tokenizer.question_encoder
       a_		    =  rag_tokenizer.generator

       if indexed_dataset is not None:
         a_		    =  '''custom'''
         a_		    =  CustomHFIndex(config.retrieval_vector_size      ,		lowercase__       )
       else:
         a_		    =  cls._build_index(lowercase__       )

       return cls(
           lowercase__      ,		question_encoder_tokenizer=lowercase__      ,		generator_tokenizer=lowercase__      ,		retrieval_workers=lowercase__      ,		index=lowercase__      ,		)



     def 				__magic_name__		(  self   :							Optional[Any]       ):
       logger.info('''re-loading the new dataset with embeddings'''       )
       # access from the training loop

       ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]       )

       # build the index object again
       a_		    =  self._build_index(self.config       )

       ray.get(
           [
               worker.create_rag_retriever.remote(
                   self.config      ,		self.question_encoder_tokenizer      ,		self.generator_tokenizer      ,		lowercase__       )
               for worker in self.retrieval_workers
           ]       )
