code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  import csv
  import json
  import os
  
  import datasets
  
  
  # TODO: Add BibTeX citation
  # Find for instance the citation on arxiv or on the dataset repo/website
  _lowerCamelCase				:	List[str]				    =       '\\n@InProceedings{huggingface:dataset,\ntitle = {A great new dataset},\nauthor={huggingface, Inc.\n},\nyear={2020}\n}\n'
  
  # TODO: Add description of the dataset here
  # You can copy an official description
  _lowerCamelCase				:	Any				    =       '\\nThis new dataset is designed to solve this great NLP task and is crafted with a lot of care.\n'
  
  # TODO: Add a link to an official homepage for the dataset here
  _lowerCamelCase				:	Union[str, Any]				    =       ''
  
  # TODO: Add the licence for the dataset here if you can find it
  _lowerCamelCase				:	Union[str, Any]				    =       ''
  
  # TODO: Add link to the official dataset URLs here
  # The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
  # This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
  _lowerCamelCase				:	int				    =       {
      'first_domain': 'https://huggingface.co/great-new-dataset-first_domain.zip',
      'second_domain': 'https://huggingface.co/great-new-dataset-second_domain.zip',
  }
  
  
  
  class 							__UpperCAmelCase  (       datasets.GeneratorBasedBuilder						):
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		__lowerCAmelCase 				=    datasets.Version('''1.1.0'''						)
  
  		# This is an example of a dataset with multiple configurations.
  		# If you don't want/need to define several sub-sets in your dataset,
  		# just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.
  
  		# If you need to make complex sub-parts in the datasets with configurable options
  		# You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
  		# BUILDER_CONFIG_CLASS = MyBuilderConfig
  
  		# You will be able to load one or the other configurations in the following list with
  		# data = datasets.load_dataset('my_dataset', 'first_domain')
  		# data = datasets.load_dataset('my_dataset', 'second_domain')
  		__lowerCAmelCase 				=    [
  		    datasets.BuilderConfig(name='''first_domain'''		,					version=A__		,					description='''This part of my dataset covers a first domain'''						),
  		    datasets.BuilderConfig(name='''second_domain'''		,					version=A__		,					description='''This part of my dataset covers a second domain'''						),
  		]
  
  		__lowerCAmelCase 				=    '''first_domain'''  # It's not mandatory to have a default configuration. Just use one if it make sense.
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[Any]      ):
  			# TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset
  			if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
  				A							  =      datasets.Features(
  				    {
  				        """sentence""": datasets.Value("""string"""      ),
  				        """option1""": datasets.Value("""string"""      ),
  				        """answer""": datasets.Value("""string"""      )
  				        # These are the features of your dataset like images, labels ...
  				    }      )
  			else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
  				A							  =      datasets.Features(
  				    {
  				        """sentence""": datasets.Value("""string"""      ),
  				        """option2""": datasets.Value("""string"""      ),
  				        """second_domain_answer""": datasets.Value("""string"""      )
  				        # These are the features of your dataset like images, labels ...
  				    }      )
  			return datasets.DatasetInfo(
  			    # This is the description that will appear on the datasets page.
  			    description=_DESCRIPTION      ,      features=_lowerCAmelCase      ,      homepage=_HOMEPAGE      ,      license=_LICENSE      ,      citation=_CITATION      ,      )
  
  
  
  
  
  
  
  		def        A     (self						:      str      ,      _lowerCAmelCase						:      List[Any]      ):
  			# TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration
  			# If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name
  
  			# dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
  			# It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
  			# By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
  			A							  =      _URLS[self.config.name]
  			A							  =      dl_manager.download_and_extract(_lowerCAmelCase      )
  			return [
  			    datasets.SplitGenerator(
  			        name=datasets.Split.TRAIN      ,      gen_kwargs={
  			            """filepath""": os.path.join(_lowerCAmelCase      ,      """train.jsonl"""      ),
  			            """split""": """train""",
  			        }      ,      ),
  			    datasets.SplitGenerator(
  			        name=datasets.Split.VALIDATION      ,      gen_kwargs={
  			            """filepath""": os.path.join(_lowerCAmelCase      ,      """dev.jsonl"""      ),
  			            """split""": """dev""",
  			        }      ,      ),
  			    datasets.SplitGenerator(
  			        name=datasets.Split.TEST      ,      gen_kwargs={
  			            """filepath""": os.path.join(_lowerCAmelCase      ,      """test.jsonl"""      ),
  			            """split""": """test"""
  			        }      ,      ),
  			]
  
  
  
  
  
  
  
  		def        A     (self						:      Any      ,      _lowerCAmelCase						:      Tuple      ,      _lowerCAmelCase						:      List[str]      ):
  			# TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
  			# The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
  			with open(_lowerCAmelCase      ,      encoding="""utf-8"""      ) as f:
  				for key, row in enumerate(_lowerCAmelCase      ):
  					A							  =      json.loads(_lowerCAmelCase      )
  					if self.config.name == "first_domain":
  						# Yields examples as (key, example) tuples
  						yield key, {
  						    "sentence": data["sentence"],
  						    "option1": data["option1"],
  						    "answer": "" if split == "test" else data["answer"],
  						}
  					else:
  						yield key, {
  						    "sentence": data["sentence"],
  						    "option2": data["option2"],
  						    "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
  						}
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  import logging
  import random
  
  import ray
  
  from transformers import RagConfig, RagRetriever, RagTokenizer
  from transformers.models.rag.retrieval_rag import CustomHFIndex
  
  
  _lowerCamelCase				:	Tuple				    =       logging.getLogger(__name__)
  
  
  
  class 							__UpperCAmelCase  :
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		def __init__(self						:      Union[str, Any]      ):
  			A							  =      False
  
  
  
  
  
  
  
  		def        A     (self						:      Any      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Any      ):
  			if not self.initialized:
  				A							  =      RagRetriever(
  				    _lowerCAmelCase      ,      question_encoder_tokenizer=_lowerCAmelCase      ,      generator_tokenizer=_lowerCAmelCase      ,      index=_lowerCAmelCase      ,      init_retrieval=_lowerCAmelCase      ,      )
  				A							  =      True
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[int]      ):
  			self.retriever.index.init_index()
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[int]      ):
  			# delete the old self.retriever object before assigning the new index
  			del self.retriever
  			A							  =      False
  
  
  
  
  
  
  
  		def        A     (self						:      List[str]      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      List[str]      ):
  			A  ,    A							  =      self.retriever._main_retrieve(_lowerCAmelCase      ,      _lowerCAmelCase      )
  			A							  =      self.retriever.index.get_doc_dicts(_lowerCAmelCase      )
  			return doc_ids, retrieved_doc_embeds, doc_dicts
  
  
  
  
  
  
  class 							__UpperCAmelCase  (       A__						):
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		def __init__(self						:      Tuple      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      Tuple      ,      _lowerCAmelCase						:      str=None      ):
  			if index is not None and index.is_initialized() and len(_lowerCAmelCase      ) > 0:
  				raise ValueError(
  				    """When using Ray for distributed fine-tuning, """
  				    """you'll need to provide the paths instead, """
  				    """as the dataset and the index are loaded """
  				    """separately. More info in examples/rag/use_own_knowledge_dataset.py """      )
  
  			super().__init__(
  			    _lowerCAmelCase      ,      question_encoder_tokenizer=_lowerCAmelCase      ,      generator_tokenizer=_lowerCAmelCase      ,      index=_lowerCAmelCase      ,      init_retrieval=_lowerCAmelCase      ,      )
  
  			A							  =      retrieval_workers
  			A							  =      question_encoder_tokenizer
  			A							  =      generator_tokenizer
  			if len(self.retrieval_workers      ) > 0:
  				ray.get(
  				    [
  				        worker.create_rag_retriever.remote(_lowerCAmelCase      ,      _lowerCAmelCase      ,      _lowerCAmelCase      ,      _lowerCAmelCase      )
  				        for worker in self.retrieval_workers
  				    ]      )
  
  
  
  
  
  
  
  		def        A     (self						:      List[Any]      ):
  			logger.info("""initializing retrieval"""      )
  
  			if len(self.retrieval_workers      ) > 0:
  				ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]      )
  			else:
  				# Non-distributed training. Load index into this same process.
  				self.index.init_index()
  
  
  
  
  
  
  
  		def        A     (self						:      str      ,      _lowerCAmelCase						:      Dict      ,      _lowerCAmelCase						:      Tuple      ):
  			if len(self.retrieval_workers      ) > 0:
  				# Select a random retrieval actor.
  				A							  =      self.retrieval_workers[random.randint(0      ,      len(self.retrieval_workers      ) - 1      )]
  				A  ,    A  ,    A							  =      ray.get(
  				    random_worker.retrieve.remote(_lowerCAmelCase      ,      _lowerCAmelCase      )      )
  			else:
  				A  ,    A							  =      self._main_retrieve(_lowerCAmelCase      ,      _lowerCAmelCase      )
  				A							  =      self.index.get_doc_dicts(_lowerCAmelCase      )
  			return retrieved_doc_embeds, doc_ids, doc_dicts
  
  
  
  
  
  
  
  		@classmethod
  		def        A     (cls						:      Union[str, Any]      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      Optional[Any]=None      ,      **_lowerCAmelCase						:      Union[str, Any]      ):
  			return super(_lowerCAmelCase      ,      cls      ).get_tokenizers(_lowerCAmelCase      ,      _lowerCAmelCase      ,      **_lowerCAmelCase      )
  
  
  
  
  
  
  
  		@classmethod
  		def        A     (cls						:      List[Any]      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      str=None      ,      **_lowerCAmelCase						:      List[str]      ):
  			A							  =      kwargs.pop("""config"""      ,      _lowerCAmelCase      ) or RagConfig.from_pretrained(_lowerCAmelCase      ,      **_lowerCAmelCase      )
  			A							  =      RagTokenizer.from_pretrained(_lowerCAmelCase      ,      config=_lowerCAmelCase      )
  			A							  =      rag_tokenizer.question_encoder
  			A							  =      rag_tokenizer.generator
  
  			if indexed_dataset is not None:
  				A							  =      """custom"""
  				A							  =      CustomHFIndex(config.retrieval_vector_size      ,      _lowerCAmelCase      )
  			else:
  				A							  =      cls._build_index(_lowerCAmelCase      )
  
  			return cls(
  			    _lowerCAmelCase      ,      question_encoder_tokenizer=_lowerCAmelCase      ,      generator_tokenizer=_lowerCAmelCase      ,      retrieval_workers=_lowerCAmelCase      ,      index=_lowerCAmelCase      ,      )
  
  
  
  
  
  
  
  		def        A     (self						:      List[Any]      ):
  			logger.info("""re-loading the new dataset with embeddings"""      )
  			# access from the training loop
  
  			ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]      )
  
  			# build the index object again
  			A							  =      self._build_index(self.config      )
  
  			ray.get(
  			    [
  			        worker.create_rag_retriever.remote(
  			            self.config      ,      self.question_encoder_tokenizer      ,      self.generator_tokenizer      ,      _lowerCAmelCase      )
  			        for worker in self.retrieval_workers
  			    ]      )
  
  