
"""simple docstring"""
import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    MarkupLMForQuestionAnswering,
			    MarkupLMForSequenceClassification,
			    MarkupLMForTokenClassification,
			    MarkupLMModel,
			)

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer




class    _snake_case  :






							def __init__(					self						:			Tuple						, snake_case__						:			Union[str, Any]						, snake_case__						:			int=1_3						, snake_case__						:			Dict=7						, snake_case__						:			str=True						, snake_case__						:			Optional[Any]=True						, snake_case__						:			int=True						, snake_case__						:			Dict=True						, snake_case__						:			List[Any]=9_9						, snake_case__						:			str=3_2						, snake_case__						:			int=5						, snake_case__						:			List[Any]=4						, snake_case__						:			Union[str, Any]=3_7						, snake_case__						:			Optional[Any]="gelu"						, snake_case__						:			List[Any]=0.1						, snake_case__						:			str=0.1						, snake_case__						:			Optional[int]=5_1_2						, snake_case__						:			List[str]=1_6						, snake_case__						:			Dict=2						, snake_case__						:			str=0.02						, snake_case__						:			Tuple=3						, snake_case__						:			Union[str, Any]=None						, snake_case__						:			Dict=2_0						, snake_case__						:			str=3_0						, snake_case__						:			List[str]=2						, snake_case__						:			List[Any]=2						, snake_case__						:			List[str]=1_0						, ):






												"""simple docstring"""



												__a:						List[Any]		    =					parent
												__a:						List[str]		    =					batch_size
												__a:						Dict		    =					seq_length
												__a:						Union[str, Any]		    =					is_training
												__a:						int		    =					use_input_mask
												__a:						str		    =					use_token_type_ids
												__a:						Optional[Any]		    =					use_labels
												__a:						Optional[Any]		    =					vocab_size
												__a:						Tuple		    =					hidden_size
												__a:						List[str]		    =					num_hidden_layers
												__a:						str		    =					num_attention_heads
												__a:						List[str]		    =					intermediate_size
												__a:						List[str]		    =					hidden_act
												__a:						str		    =					hidden_dropout_prob
												__a:						Dict		    =					attention_probs_dropout_prob
												__a:						int		    =					max_position_embeddings
												__a:						Tuple		    =					type_vocab_size
												__a:						Optional[int]		    =					type_sequence_label_size
												__a:						Union[str, Any]		    =					initializer_range
												__a:						Union[str, Any]		    =					num_labels
												__a:						List[str]		    =					scope
												__a:						List[Any]		    =					max_xpath_tag_unit_embeddings
												__a:						Tuple		    =					max_xpath_subs_unit_embeddings
												__a:						Any		    =					tag_pad_id
												__a:						Any		    =					subs_pad_id
												__a:						Any		    =					max_depth






							def 							__lowercase  (					self						:			List[str]    ):






												"""simple docstring"""



												__a:						List[str]		    =					ids_tensor([self.batch_size, self.seq_length]						, self.vocab_size    )

												__a:						Union[str, Any]		    =					ids_tensor(
												    [self.batch_size, self.seq_length, self.max_depth]						, self.max_xpath_tag_unit_embeddings    )

												__a:						List[Any]		    =					ids_tensor(
												    [self.batch_size, self.seq_length, self.max_depth]						, self.max_xpath_subs_unit_embeddings    )

												__a:						Union[str, Any]		    =					None
												if self.use_input_mask:
																	__a:						Any		    =					ids_tensor([self.batch_size, self.seq_length]						, vocab_size=2    )

												__a:						Any		    =					None
												if self.use_token_type_ids:
																	__a:						List[Any]		    =					ids_tensor([self.batch_size, self.seq_length]						, self.type_vocab_size    )

												__a:						Tuple		    =					None
												__a:						Dict		    =					None
												if self.use_labels:
																	__a:						Optional[int]		    =					ids_tensor([self.batch_size]						, self.type_sequence_label_size    )
																	__a:						Any		    =					ids_tensor([self.batch_size, self.seq_length]						, self.num_labels    )

												__a:						List[str]		    =					self.get_config()

												return (
												    config,
												    input_ids,
												    xpath_tags_seq,
												    xpath_subs_seq,
												    token_type_ids,
												    input_mask,
												    sequence_labels,
												    token_labels,
												)






							def 							__lowercase  (					self						:			Optional[Any]    ):






												"""simple docstring"""



												return MarkupLMConfig(
												    vocab_size=self.vocab_size						, hidden_size=self.hidden_size						, num_hidden_layers=self.num_hidden_layers						, num_attention_heads=self.num_attention_heads						, intermediate_size=self.intermediate_size						, hidden_act=self.hidden_act						, hidden_dropout_prob=self.hidden_dropout_prob						, attention_probs_dropout_prob=self.attention_probs_dropout_prob						, max_position_embeddings=self.max_position_embeddings						, type_vocab_size=self.type_vocab_size						, initializer_range=self.initializer_range						, max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings						, max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings						, tag_pad_id=self.tag_pad_id						, subs_pad_id=self.subs_pad_id						, max_depth=self.max_depth						, )






							def 							__lowercase  (					self						:			Tuple						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			str						, snake_case__						:			Dict						, snake_case__						:			Dict						, snake_case__						:			int						, snake_case__						:			List[str]						, snake_case__						:			str						, ):






												"""simple docstring"""



												__a:						List[str]		    =					MarkupLMModel(config=snake_case__    )
												model.to(snake_case__    )
												model.eval()
												print("""Configs:"""						, model.config.tag_pad_id						, model.config.subs_pad_id    )
												__a:						Any		    =					model(snake_case__						, attention_mask=snake_case__						, token_type_ids=snake_case__    )
												__a:						List[Any]		    =					model(snake_case__						, token_type_ids=snake_case__    )
												__a:						Optional[int]		    =					model(snake_case__    )
												self.parent.assertEqual(result.last_hidden_state.shape						, (self.batch_size, self.seq_length, self.hidden_size)    )
												self.parent.assertEqual(result.pooler_output.shape						, (self.batch_size, self.hidden_size)    )






							def 							__lowercase  (					self						:			Tuple						, snake_case__						:			Optional[Any]						, snake_case__						:			Any						, snake_case__						:			List[Any]						, snake_case__						:			List[Any]						, snake_case__						:			Any						, snake_case__						:			Optional[Any]						, snake_case__						:			List[str]						, snake_case__						:			Optional[Any]						, ):






												"""simple docstring"""



												__a:						List[str]		    =					self.num_labels
												__a:						int		    =					MarkupLMForSequenceClassification(snake_case__    )
												model.to(snake_case__    )
												model.eval()
												__a:						str		    =					model(
												    snake_case__						, xpath_tags_seq=snake_case__						, xpath_subs_seq=snake_case__						, attention_mask=snake_case__						, token_type_ids=snake_case__						, labels=snake_case__						, )
												self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.num_labels)    )






							def 							__lowercase  (					self						:			Dict						, snake_case__						:			Optional[Any]						, snake_case__						:			List[Any]						, snake_case__						:			Optional[Any]						, snake_case__						:			Union[str, Any]						, snake_case__						:			Dict						, snake_case__						:			int						, snake_case__						:			Union[str, Any]						, snake_case__						:			Any						, ):






												"""simple docstring"""



												__a:						str		    =					self.num_labels
												__a:						List[Any]		    =					MarkupLMForTokenClassification(config=snake_case__    )
												model.to(snake_case__    )
												model.eval()
												__a:						Dict		    =					model(
												    snake_case__						, xpath_tags_seq=snake_case__						, xpath_subs_seq=snake_case__						, attention_mask=snake_case__						, token_type_ids=snake_case__						, labels=snake_case__						, )
												self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length, self.num_labels)    )






							def 							__lowercase  (					self						:			Tuple						, snake_case__						:			Dict						, snake_case__						:			Any						, snake_case__						:			str						, snake_case__						:			List[Any]						, snake_case__						:			Optional[int]						, snake_case__						:			List[str]						, snake_case__						:			Union[str, Any]						, snake_case__						:			int						, ):






												"""simple docstring"""



												__a:						Union[str, Any]		    =					MarkupLMForQuestionAnswering(config=snake_case__    )
												model.to(snake_case__    )
												model.eval()
												__a:						Dict		    =					model(
												    snake_case__						, xpath_tags_seq=snake_case__						, xpath_subs_seq=snake_case__						, attention_mask=snake_case__						, token_type_ids=snake_case__						, start_positions=snake_case__						, end_positions=snake_case__						, )
												self.parent.assertEqual(result.start_logits.shape						, (self.batch_size, self.seq_length)    )
												self.parent.assertEqual(result.end_logits.shape						, (self.batch_size, self.seq_length)    )






							def 							__lowercase  (					self						:			Optional[Any]    ):






												"""simple docstring"""



												__a:						int		    =					self.prepare_config_and_inputs()
												(
												    (
												    __a       
												)    ,				(
												    __a       
												)    ,				(
												    __a       
												)    ,				(
												    __a       
												)    ,				(
												    __a       
												)    ,				(
												    __a       
												)    ,				(
												    __a       
												)    ,				(
												    __a       
												)    ,				       
												):						Tuple		    =					config_and_inputs
												__a:						Union[str, Any]		    =					{
												    """input_ids""": input_ids,
												    """xpath_tags_seq""": xpath_tags_seq,
												    """xpath_subs_seq""": xpath_subs_seq,
												    """token_type_ids""": token_type_ids,
												    """attention_mask""": input_mask,
												}
												return config, inputs_dict






@require_torch
class    _snake_case  (						_UpperCAmelCase ,				_UpperCAmelCase ,				unittest.TestCase       ):
							SCREAMING_SNAKE_CASE		     =						(
							    (
							        MarkupLMModel,
							        MarkupLMForSequenceClassification,
							        MarkupLMForTokenClassification,
							        MarkupLMForQuestionAnswering,
							    )
							    if is_torch_available()
							    else None
							)
							SCREAMING_SNAKE_CASE		     =						(
							    {
							        '''feature-extraction''': MarkupLMModel,
							        '''question-answering''': MarkupLMForQuestionAnswering,
							        '''text-classification''': MarkupLMForSequenceClassification,
							        '''token-classification''': MarkupLMForTokenClassification,
							        '''zero-shot''': MarkupLMForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)






							def 							__lowercase  (					self						:			Tuple						, snake_case__						:			int						, snake_case__						:			List[str]						, snake_case__						:			List[str]						, snake_case__						:			Dict						, snake_case__						:			Dict    ):






												"""simple docstring"""



												return True






							def 							__lowercase  (					self						:			Any    ):






												"""simple docstring"""



												__a:						Any		    =					MarkupLMModelTester(self    )
												__a:						List[str]		    =					ConfigTester(self						, config_class=snake_case__						, hidden_size=3_7    )






							def 							__lowercase  (					self						:			Tuple    ):






												"""simple docstring"""



												self.config_tester.run_common_tests()






							def 							__lowercase  (					self						:			Any    ):






												"""simple docstring"""



												__a:						Union[str, Any]		    =					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*snake_case__    )






							def 							__lowercase  (					self						:			List[Any]    ):






												"""simple docstring"""



												__a:						List[Any]		    =					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*snake_case__    )






							def 							__lowercase  (					self						:			Dict    ):






												"""simple docstring"""



												__a:						Tuple		    =					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*snake_case__    )






							def 							__lowercase  (					self						:			Any    ):






												"""simple docstring"""



												__a:						List[str]		    =					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*snake_case__    )






def     lowerCamelCase_    (	)     ->			Optional[int]:


					'''simple docstring'''
					__a:						List[str]		    =					"""
    <!DOCTYPE html>
    <html>
    <head>
    <title>Page Title</title>
    </head>
    <body>

    <h1>This is a Heading</h1>
    <p>This is a paragraph.</p>

    </body>
    </html>
    """

					return html_string




@require_torch
class    _snake_case  (						unittest.TestCase       ):






							@cached_property
							def 							__lowercase  (					self						:			Optional[Any]    ):






												"""simple docstring"""



												__a:						List[str]		    =					MarkupLMFeatureExtractor()
												__a:						Optional[Any]		    =					MarkupLMTokenizer.from_pretrained("""microsoft/markuplm-base"""    )

												return MarkupLMProcessor(snake_case__						, snake_case__    )






							@slow
							def 							__lowercase  (					self						:			Union[str, Any]    ):






												"""simple docstring"""



												__a:						Dict		    =					MarkupLMModel.from_pretrained("""microsoft/markuplm-base"""    ).to(snake_case__    )

												__a:						List[Any]		    =					self.default_processor

												__a:						int		    =					processor(prepare_html_string()						, return_tensors="""pt"""    )
												__a:						List[str]		    =					inputs.to(snake_case__    )

												# forward pass
												with torch.no_grad():
																	__a:						Optional[Any]		    =					model(**snake_case__    )

												# verify the last hidden states
												__a:						Optional[Any]		    =					torch.Size([1, 1_4, 7_6_8]    )
												self.assertEqual(outputs.last_hidden_state.shape						, snake_case__    )

												__a:						Dict		    =					torch.tensor(
												    [[0.0_675, -0.0_052, 0.5_001], [-0.2_281, 0.0_802, 0.2_192], [-0.0_583, -0.3_311, 0.1_185]]    ).to(snake_case__    )

												self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]						, snake_case__						, atol=1e-4    )    )


"""simple docstring"""
import json
import os
import shutil
import sys
import tempfile
import unittest
from unittest import TestCase
from unittest.mock import patch

import faiss
import numpy as np
from datasets import Dataset

from transformers import BartConfig, BartTokenizer, DPRConfig, DPRQuestionEncoderTokenizer, RagConfig
from transformers.file_utils import is_datasets_available, is_faiss_available, is_psutil_available, is_torch_available
from transformers.integrations import is_ray_available
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_ray


sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip

if is_torch_available():
			from distributed_pytorch_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
else:
			lowercase		:						str =					None

if is_ray_available():
			import ray  # noqa: E402 # isort:skip
			from distributed_ray_retriever import RagRayDistributedRetriever, RayRetriever  # noqa: E402 # isort:skip
else:
			lowercase		:						Optional[int] =					None
			lowercase		:						Union[str, Any] =					None
			lowercase		:						List[str] =					None


def     lowerCamelCase_    (	lowerCamelCase_ )     ->			Union[str, Any]:


					'''simple docstring'''
					if not (is_datasets_available() and is_faiss_available() and is_psutil_available()):
										__a:						int		    =					unittest.skip("""test requires Datasets, Faiss, psutil""" )(lowerCamelCase_ )
					return test_case




@require_distributed_retrieval
class    _snake_case  (						_lowercase       ):






							def 							__lowercase  (					self						:			Any    ):






												"""simple docstring"""



												__a:						Optional[Any]		    =					tempfile.mkdtemp()
												__a:						int		    =					8

												# DPR tok
												__a:						Any		    =					[
												    """[UNK]""",
												    """[CLS]""",
												    """[SEP]""",
												    """[PAD]""",
												    """[MASK]""",
												    """want""",
												    """##want""",
												    """##ed""",
												    """wa""",
												    """un""",
												    """runn""",
												    """##ing""",
												    """,""",
												    """low""",
												    """lowest""",
												]
												__a:						str		    =					os.path.join(self.tmpdirname						, """dpr_tokenizer"""    )
												os.makedirs(snake_case__						, exist_ok=snake_case__    )
												__a:						Tuple		    =					os.path.join(snake_case__						, DPR_VOCAB_FILES_NAMES["""vocab_file"""]    )
												with open(self.vocab_file						, """w"""						, encoding="""utf-8"""    ) as vocab_writer:
																	vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )

												# BART tok
												__a:						Any		    =					[
												    """l""",
												    """o""",
												    """w""",
												    """e""",
												    """r""",
												    """s""",
												    """t""",
												    """i""",
												    """d""",
												    """n""",
												    """\u0120""",
												    """\u0120l""",
												    """\u0120n""",
												    """\u0120lo""",
												    """\u0120low""",
												    """er""",
												    """\u0120lowest""",
												    """\u0120newer""",
												    """\u0120wider""",
												    """<unk>""",
												]
												__a:						List[str]		    =					dict(zip(snake_case__						, range(len(snake_case__    )    )    )    )
												__a:						List[str]		    =					["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
												__a:						List[str]		    =					{"""unk_token""": """<unk>"""}

												__a:						Optional[int]		    =					os.path.join(self.tmpdirname						, """bart_tokenizer"""    )
												os.makedirs(snake_case__						, exist_ok=snake_case__    )
												__a:						Tuple		    =					os.path.join(snake_case__						, BART_VOCAB_FILES_NAMES["""vocab_file"""]    )
												__a:						str		    =					os.path.join(snake_case__						, BART_VOCAB_FILES_NAMES["""merges_file"""]    )
												with open(self.vocab_file						, """w"""						, encoding="""utf-8"""    ) as fp:
																	fp.write(json.dumps(snake_case__    ) + """\n"""    )
												with open(self.merges_file						, """w"""						, encoding="""utf-8"""    ) as fp:
																	fp.write("""\n""".join(snake_case__    )    )






							def 							__lowercase  (					self						:			Any    ):






												"""simple docstring"""



												return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname						, """dpr_tokenizer"""    )    )






							def 							__lowercase  (					self						:			Any    ):






												"""simple docstring"""



												return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname						, """bart_tokenizer"""    )    )






							def 							__lowercase  (					self						:			Tuple    ):






												"""simple docstring"""



												shutil.rmtree(self.tmpdirname    )






							def 							__lowercase  (					self						:			int    ):






												"""simple docstring"""



												__a:						int		    =					Dataset.from_dict(
												    {
												        """id""": ["""0""", """1"""],
												        """text""": ["""foo""", """bar"""],
												        """title""": ["""Foo""", """Bar"""],
												        """embeddings""": [np.ones(self.retrieval_vector_size    ), 2 * np.ones(self.retrieval_vector_size    )],
												    }    )
												dataset.add_faiss_index("""embeddings"""						, string_factory="""Flat"""						, metric_type=faiss.METRIC_INNER_PRODUCT    )
												return dataset






							def 							__lowercase  (					self						:			Any						, snake_case__						:			bool						, snake_case__						:			Dict=1_2_3_4_5    ):






												"""simple docstring"""



												__a:						str		    =					self.get_dummy_dataset()
												__a:						int		    =					RagConfig(
												    retrieval_vector_size=self.retrieval_vector_size						, question_encoder=DPRConfig().to_dict()						, generator=BartConfig().to_dict()						, )
												with patch("""transformers.models.rag.retrieval_rag.load_dataset"""    ) as mock_load_dataset:
																	__a:						List[str]		    =					dataset
																	__a:						List[Any]		    =					RagPyTorchDistributedRetriever(
																	    snake_case__						, question_encoder_tokenizer=self.get_dpr_tokenizer()						, generator_tokenizer=self.get_bart_tokenizer()						, )
																	if init_retrieval:
																						retriever.init_retrieval(snake_case__    )
												return retriever






							def 							__lowercase  (					self						:			Dict						, snake_case__						:			bool    ):






												"""simple docstring"""



												ray.init(local_mode=snake_case__    )
												__a:						int		    =					RagConfig(
												    retrieval_vector_size=self.retrieval_vector_size						, question_encoder=DPRConfig().to_dict()						, generator=BartConfig().to_dict()						, )
												__a:						List[str]		    =					ray.remote(snake_case__    )
												__a:						Tuple		    =					[remote_cls.remote() for _ in range(1    )]
												with patch("""transformers.models.rag.retrieval_rag.load_dataset"""    ) as mock_load_dataset:
																	__a:						int		    =					self.get_dummy_dataset()
																	__a:						List[Any]		    =					RagRayDistributedRetriever(
																	    snake_case__						, question_encoder_tokenizer=self.get_dpr_tokenizer()						, generator_tokenizer=self.get_bart_tokenizer()						, retrieval_workers=snake_case__						, )
																	if init_retrieval:
																						retriever.init_retrieval()
												return retriever






							def 							__lowercase  (					self						:			str						, snake_case__						:			bool						, snake_case__						:			bool						, snake_case__						:			Union[str, Any]=1_2_3_4_5    ):






												"""simple docstring"""



												__a:						List[Any]		    =					self.get_dummy_dataset()
												__a:						Optional[Any]		    =					RagConfig(
												    retrieval_vector_size=self.retrieval_vector_size						, question_encoder=DPRConfig().to_dict()						, generator=BartConfig().to_dict()						, index_name="""custom"""						, )
												if from_disk:
																	__a:						int		    =					os.path.join(self.tmpdirname						, """dataset"""    )
																	__a:						int		    =					os.path.join(self.tmpdirname						, """index.faiss"""    )
																	dataset.get_index("""embeddings"""    ).save(os.path.join(self.tmpdirname						, """index.faiss"""    )    )
																	dataset.drop_index("""embeddings"""    )
																	dataset.save_to_disk(os.path.join(self.tmpdirname						, """dataset"""    )    )
																	del dataset
																	__a:						str		    =					RagPyTorchDistributedRetriever(
																	    snake_case__						, question_encoder_tokenizer=self.get_dpr_tokenizer()						, generator_tokenizer=self.get_bart_tokenizer()						, )
												else:
																	__a:						List[Any]		    =					RagPyTorchDistributedRetriever(
																	    snake_case__						, question_encoder_tokenizer=self.get_dpr_tokenizer()						, generator_tokenizer=self.get_bart_tokenizer()						, index=CustomHFIndex(config.retrieval_vector_size						, snake_case__    )						, )
												if init_retrieval:
																	retriever.init_retrieval(snake_case__    )
												return retriever






							def 							__lowercase  (					self						:			str						, snake_case__						:			bool						, snake_case__						:			bool    ):






												"""simple docstring"""



												ray.init(local_mode=snake_case__    )
												__a:						Tuple		    =					self.get_dummy_dataset()
												__a:						Dict		    =					RagConfig(
												    retrieval_vector_size=self.retrieval_vector_size						, question_encoder=DPRConfig().to_dict()						, generator=BartConfig().to_dict()						, index_name="""custom"""						, )
												__a:						Optional[Any]		    =					ray.remote(snake_case__    )
												__a:						Any		    =					[remote_cls.remote() for _ in range(1    )]
												if from_disk:
																	__a:						List[Any]		    =					os.path.join(self.tmpdirname						, """dataset"""    )
																	__a:						Any		    =					os.path.join(self.tmpdirname						, """index.faiss"""    )
																	dataset.get_index("""embeddings"""    ).save(os.path.join(self.tmpdirname						, """index.faiss"""    )    )
																	dataset.drop_index("""embeddings"""    )
																	dataset.save_to_disk(os.path.join(self.tmpdirname						, """dataset"""    )    )
																	del dataset
																	__a:						Any		    =					RagRayDistributedRetriever(
																	    snake_case__						, question_encoder_tokenizer=self.get_dpr_tokenizer()						, generator_tokenizer=self.get_bart_tokenizer()						, retrieval_workers=snake_case__						, index=CustomHFIndex.load_from_disk(
																	        vector_size=config.retrieval_vector_size						, dataset_path=config.passages_path						, index_path=config.index_path						, )						, )
												else:
																	__a:						Any		    =					RagRayDistributedRetriever(
																	    snake_case__						, question_encoder_tokenizer=self.get_dpr_tokenizer()						, generator_tokenizer=self.get_bart_tokenizer()						, retrieval_workers=snake_case__						, index=CustomHFIndex(config.retrieval_vector_size						, snake_case__    )						, )
												if init_retrieval:
																	retriever.init_retrieval()
												return retriever






							def 							__lowercase  (					self						:			str						, snake_case__						:			RagRetriever						, snake_case__						:			np.array						, snake_case__						:			int    ):






												"""simple docstring"""



												__a    ,				__a    ,				__a:						Optional[Any]		    =					retriever.retrieve(snake_case__						, n_docs=snake_case__    )
												self.assertEqual(retrieved_doc_embeds.shape						, (2, n_docs, self.retrieval_vector_size)    )
												self.assertEqual(len(snake_case__    )						, 2    )
												self.assertEqual(sorted(doc_dicts[0]    )						, ["""embeddings""", """id""", """text""", """title"""]    )
												self.assertEqual(len(doc_dicts[0]["""id"""]    )						, snake_case__    )
												self.assertEqual(doc_dicts[0]["""id"""][0]						, """1"""    )  # max inner product is reached with second doc
												self.assertEqual(doc_dicts[1]["""id"""][0]						, """0"""    )  # max inner product is reached with first doc
												self.assertListEqual(doc_ids.tolist()						, [[1], [0]]    )






							def 							__lowercase  (					self						:			List[str]    ):






												"""simple docstring"""



												__a:						int		    =					1
												__a:						Tuple		    =					np.array(
												    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]						, dtype=np.floataa    )

												self.distributed_retriever_check(
												    self.get_dummy_pytorch_distributed_retriever(init_retrieval=snake_case__    )						, snake_case__						, snake_case__    )






							def 							__lowercase  (					self						:			str    ):






												"""simple docstring"""



												__a:						Tuple		    =					1
												__a:						Tuple		    =					np.array(
												    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]						, dtype=np.floataa    )

												self.distributed_retriever_check(
												    self.get_dummy_custom_hf_index_pytorch_retriever(init_retrieval=snake_case__						, from_disk=snake_case__    )						, snake_case__						, snake_case__						, )






							def 							__lowercase  (					self						:			List[Any]    ):






												"""simple docstring"""



												__a:						List[Any]		    =					1
												__a:						int		    =					np.array(
												    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]						, dtype=np.floataa    )

												self.distributed_retriever_check(
												    self.get_dummy_custom_hf_index_pytorch_retriever(init_retrieval=snake_case__						, from_disk=snake_case__    )						, snake_case__						, snake_case__						, )






							@require_ray
							def 							__lowercase  (					self						:			str    ):






												"""simple docstring"""



												__a:						int		    =					1
												__a:						str		    =					np.array(
												    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]						, dtype=np.floataa    )

												self.distributed_retriever_check(
												    self.get_dummy_ray_distributed_retriever(init_retrieval=snake_case__    )						, snake_case__						, snake_case__    )
												ray.shutdown()






							@require_ray
							def 							__lowercase  (					self						:			Dict    ):






												"""simple docstring"""



												__a:						List[Any]		    =					1
												__a:						List[str]		    =					np.array(
												    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]						, dtype=np.floataa    )
												with self.assertRaises(snake_case__    ):
																	self.distributed_retriever_check(
																	    self.get_dummy_custom_hf_index_ray_retriever(init_retrieval=snake_case__						, from_disk=snake_case__    )						, snake_case__						, snake_case__						, )
												ray.shutdown()






							@require_ray
							def 							__lowercase  (					self						:			List[str]    ):






												"""simple docstring"""



												__a:						Tuple		    =					1
												__a:						str		    =					np.array(
												    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]						, dtype=np.floataa    )

												self.distributed_retriever_check(
												    self.get_dummy_custom_hf_index_ray_retriever(init_retrieval=snake_case__						, from_disk=snake_case__    )						, snake_case__						, snake_case__    )
												ray.shutdown()
