





'''simple docstring'''


import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


lowerCAmelCase      :  Union[str, Any]      							=logging.getLogger(__name__)
class    a_     :





     def __init__(				self	:      Optional[Any]		):





        """simple docstring"""




        lowercase_    :int					    =				False





     def 							lowercase__   (				self	:      int			,   lowercase	:      Optional[int]			,   lowercase	:      Optional[Any]			,   lowercase	:      Any			,   lowercase	:      List[str]		):





        """simple docstring"""




        if not self.initialized:
           lowercase_    :str					    =				RagRetriever(
               lowercase			,   question_encoder_tokenizer=lowercase			,   generator_tokenizer=lowercase			,   index=lowercase			,   init_retrieval=lowercase			,   )
           lowercase_    :Optional[Any]					    =				True





     def 							lowercase__   (				self	:      Tuple		):





        """simple docstring"""




        self.retriever.index.init_index()





     def 							lowercase__   (				self	:      Optional[Any]		):





        """simple docstring"""




        del self.retriever
        lowercase_    :Any					    =				False







     def 							lowercase__   (				self	:      str			,   lowercase	:      Tuple			,   lowercase	:      Tuple		):





        """simple docstring"""




        lowercase_    ,							lowercase_    :Any					    =				self.retriever._main_retrieve(lowercase			,   lowercase		)
        lowercase_    :List[Any]					    =				self.retriever.index.get_doc_dicts(lowercase		)
        return doc_ids, retrieved_doc_embeds, doc_dicts







class    a_     (       _lowerCAmelCase							):





     def __init__(				self	:      Optional[Any]			,   lowercase	:      List[str]			,   lowercase	:      List[Any]			,   lowercase	:      List[str]			,   lowercase	:      Dict			,   lowercase	:      Optional[int]=None		):





        """simple docstring"""




        if index is not None and index.is_initialized() and len(lowercase		) > 0:
           raise ValueError(
               "When using Ray for distributed fine-tuning, "
               "you'll need to provide the paths instead, "
               "as the dataset and the index are loaded "
               "separately. More info in examples/rag/use_own_knowledge_dataset.py "		)

        super().__init__(
            lowercase			,   question_encoder_tokenizer=lowercase			,   generator_tokenizer=lowercase			,   index=lowercase			,   init_retrieval=lowercase			,   )

        lowercase_    :int					    =				retrieval_workers
        lowercase_    :Any					    =				question_encoder_tokenizer
        lowercase_    :List[Any]					    =				generator_tokenizer
        if len(self.retrieval_workers		) > 0:
           ray.get(
               [
                   worker.create_rag_retriever.remote(lowercase			,   lowercase			,   lowercase			,   lowercase		)
                   for worker in self.retrieval_workers
               ]		)





     def 							lowercase__   (				self	:      int		):





        """simple docstring"""




        logger.info("initializing retrieval"		)

        if len(self.retrieval_workers		) > 0:
           ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]		)
        else:
           # Non-distributed training. Load index into this same process.
           self.index.init_index()





     def 							lowercase__   (				self	:      Optional[Any]			,   lowercase	:      Any			,   lowercase	:      Union[str, Any]		):





        """simple docstring"""




        if len(self.retrieval_workers		) > 0:
           # Select a random retrieval actor.
           lowercase_    :Tuple					    =				self.retrieval_workers[random.randint(0			,   len(self.retrieval_workers		) - 1		)]
           lowercase_    ,							lowercase_    ,							lowercase_    :Optional[int]					    =				ray.get(
               random_worker.retrieve.remote(lowercase			,   lowercase		)		)
        else:
           lowercase_    ,							lowercase_    :Optional[int]					    =				self._main_retrieve(lowercase			,   lowercase		)
           lowercase_    :Optional[int]					    =				self.index.get_doc_dicts(lowercase		)
        return retrieved_doc_embeds, doc_ids, doc_dicts





     @classmethod
     def 							lowercase__   (				cls	:      Union[str, Any]			,   lowercase	:      List[Any]			,   lowercase	:      Union[str, Any]=None			,   **lowercase	:      Tuple		):





        """simple docstring"""




        return super(lowercase			,   cls		).get_tokenizers(lowercase			,   lowercase			,   **lowercase		)





     @classmethod
     def 							lowercase__   (				cls	:      Optional[Any]			,   lowercase	:      int			,   lowercase	:      Union[str, Any]			,   lowercase	:      Optional[Any]=None			,   **lowercase	:      List[str]		):





        """simple docstring"""




        lowercase_    :Union[str, Any]					    =				kwargs.pop("config"			,   lowercase		) or RagConfig.from_pretrained(lowercase			,   **lowercase		)
        lowercase_    :Tuple					    =				RagTokenizer.from_pretrained(lowercase			,   config=lowercase		)
        lowercase_    :Tuple					    =				rag_tokenizer.question_encoder
        lowercase_    :Tuple					    =				rag_tokenizer.generator

        if indexed_dataset is not None:
           lowercase_    :int					    =				"custom"
           lowercase_    :Any					    =				CustomHFIndex(config.retrieval_vector_size			,   lowercase		)
        else:
           lowercase_    :Optional[int]					    =				cls._build_index(lowercase		)

        return cls(
            lowercase			,   question_encoder_tokenizer=lowercase			,   generator_tokenizer=lowercase			,   retrieval_workers=lowercase			,   index=lowercase			,   )







     def 							lowercase__   (				self	:      Tuple		):





        """simple docstring"""




        logger.info("re-loading the new dataset with embeddings"		)
        # access from the training loop

        ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]		)

        # build the index object again
        lowercase_    :Tuple					    =				self._build_index(self.config		)

        ray.get(
            [
                worker.create_rag_retriever.remote(
                    self.config			,   self.question_encoder_tokenizer			,   self.generator_tokenizer			,   lowercase		)
                for worker in self.retrieval_workers
            ]		)









'''simple docstring'''


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import EfficientNetImageProcessor
class    a_     (       unittest.TestCase							):





     def __init__(				self	:      int			,   lowercase	:      Optional[int]			,   lowercase	:      Tuple=13			,   lowercase	:      str=3			,   lowercase	:      List[str]=18			,   lowercase	:      Union[str, Any]=30			,   lowercase	:      Any=400			,   lowercase	:      Tuple=True			,   lowercase	:      Union[str, Any]=None			,   lowercase	:      Optional[Any]=True			,   lowercase	:      Optional[int]=[0.5, 0.5, 0.5]			,   lowercase	:      List[str]=[0.5, 0.5, 0.5]			,   ):





        """simple docstring"""




        lowercase_    :Dict					    =				size if size is not None else {"height": 18, "width": 18}
        lowercase_    :Optional[Any]					    =				parent
        lowercase_    :str					    =				batch_size
        lowercase_    :Optional[int]					    =				num_channels
        lowercase_    :str					    =				image_size
        lowercase_    :Tuple					    =				min_resolution
        lowercase_    :Optional[Any]					    =				max_resolution
        lowercase_    :List[str]					    =				do_resize
        lowercase_    :Optional[Any]					    =				size
        lowercase_    :Optional[int]					    =				do_normalize
        lowercase_    :Union[str, Any]					    =				image_mean
        lowercase_    :Dict					    =				image_std







     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        return {
            "image_mean": self.image_mean,
            "image_std": self.image_std,
            "do_normalize": self.do_normalize,
            "do_resize": self.do_resize,
            "size": self.size,
        }







@require_torch
@require_vision
class    a_     (       _lowerCAmelCase			,		unittest.TestCase							):
     __A  				=    EfficientNetImageProcessor if is_vision_available() else None





     def 							lowercase__   (				self	:      Optional[int]		):





        """simple docstring"""




        lowercase_    :List[Any]					    =				EfficientNetImageProcessorTester(self		)





     @property
     def 							lowercase__   (				self	:      Optional[Any]		):





        """simple docstring"""




        return self.image_processor_tester.prepare_image_processor_dict()





     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :Optional[Any]					    =				self.image_processing_class(**self.image_processor_dict		)
        self.assertTrue(hasattr(lowercase			,   "image_mean"		)		)
        self.assertTrue(hasattr(lowercase			,   "image_std"		)		)
        self.assertTrue(hasattr(lowercase			,   "do_normalize"		)		)
        self.assertTrue(hasattr(lowercase			,   "do_resize"		)		)
        self.assertTrue(hasattr(lowercase			,   "size"		)		)





     def 							lowercase__   (				self	:      str		):





        """simple docstring"""




        lowercase_    :Tuple					    =				self.image_processing_class.from_dict(self.image_processor_dict		)
        self.assertEqual(image_processor.size			,   {"height": 18, "width": 18}		)

        lowercase_    :List[str]					    =				self.image_processing_class.from_dict(self.image_processor_dict			,   size=42		)
        self.assertEqual(image_processor.size			,   {"height": 42, "width": 42}		)





     def 							lowercase__   (				self	:      Optional[int]		):





        """simple docstring"""




        lowercase_    :List[Any]					    =				self.image_processing_class(**self.image_processor_dict		)
        # create random PIL images
        lowercase_    :Tuple					    =				prepare_image_inputs(self.image_processor_tester			,   equal_resolution=lowercase		)
        for image in image_inputs:
           self.assertIsInstance(lowercase			,   Image.Image		)

        # Test not batched input
        lowercase_    :Optional[Any]					    =				image_processing(image_inputs[0]			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )

        # Test batched
        lowercase_    :Optional[Any]					    =				image_processing(lowercase			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )





     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :int					    =				self.image_processing_class(**self.image_processor_dict		)
        # create random numpy tensors
        lowercase_    :List[Any]					    =				prepare_image_inputs(self.image_processor_tester			,   equal_resolution=lowercase			,   numpify=lowercase		)
        for image in image_inputs:
           self.assertIsInstance(lowercase			,   np.ndarray		)

        # Test not batched input
        lowercase_    :Optional[Any]					    =				image_processing(image_inputs[0]			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )

        # Test batched
        lowercase_    :Any					    =				image_processing(lowercase			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )







     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :str					    =				self.image_processing_class(**self.image_processor_dict		)
        # create random PyTorch tensors
        lowercase_    :Tuple					    =				prepare_image_inputs(self.image_processor_tester			,   equal_resolution=lowercase			,   torchify=lowercase		)
        for image in image_inputs:
           self.assertIsInstance(lowercase			,   torch.Tensor		)

        # Test not batched input
        lowercase_    :str					    =				image_processing(image_inputs[0]			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )

        # Test batched
        lowercase_    :Optional[int]					    =				image_processing(lowercase			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )


