






import os
from argparse import ArgumentParser, Namespace

from ..data import SingleSentenceClassificationProcessor as Processor
from ..pipelines import TextClassificationPipeline
from ..utils import is_tf_available, is_torch_available, logging
from . import BaseTransformersCLICommand


if not is_tf_available() and not is_torch_available():
 raise RuntimeError('''At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training''')

# TF training parameters
lowerCamelCase	:							Dict				=False
lowerCamelCase	:							str				=False







def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase			) ->	int:
     return TrainCommand(__lowerCAmelCase			)





class 			__a  (    A__       ):







 @staticmethod
 def 					__lowercase					(		SCREAMING_SNAKE_CASE					:   ArgumentParser						):


      '''simple docstring'''






      UpperCamelCase__			:   List[Any]       				=  parser.add_parser("train"							,   help="CLI tool to train a model on a task."						)

      train_parser.add_argument(
          "--train_data"							,   type=SCREAMING_SNAKE_CASE							,   required=SCREAMING_SNAKE_CASE							,   help="path to train (and optionally evaluation) dataset as a csv with tab separated labels and sentences."							,   )
      train_parser.add_argument(
          "--column_label"							,   type=SCREAMING_SNAKE_CASE							,   default=0							,   help="Column of the dataset csv file with example labels."						)
      train_parser.add_argument(
          "--column_text"							,   type=SCREAMING_SNAKE_CASE							,   default=1							,   help="Column of the dataset csv file with example texts."						)
      train_parser.add_argument(
          "--column_id"							,   type=SCREAMING_SNAKE_CASE							,   default=2							,   help="Column of the dataset csv file with example ids."						)
      train_parser.add_argument(
          "--skip_first_row"							,   action="store_true"							,   help="Skip the first row of the csv file (headers)."						)

      train_parser.add_argument("--validation_data"							,   type=SCREAMING_SNAKE_CASE							,   default=""							,   help="path to validation dataset."						)
      train_parser.add_argument(
          "--validation_split"							,   type=SCREAMING_SNAKE_CASE							,   default=0.1							,   help="if validation dataset is not provided, fraction of train dataset to use as validation dataset."							,   )

      train_parser.add_argument("--output"							,   type=SCREAMING_SNAKE_CASE							,   default="./"							,   help="path to saved the trained model."						)

      train_parser.add_argument(
          "--task"							,   type=SCREAMING_SNAKE_CASE							,   default="text_classification"							,   help="Task to train the model on."						)
      train_parser.add_argument(
          "--model"							,   type=SCREAMING_SNAKE_CASE							,   default="bert-base-uncased"							,   help="Model's name or path to stored model."						)
      train_parser.add_argument("--train_batch_size"							,   type=SCREAMING_SNAKE_CASE							,   default=32							,   help="Batch size for training."						)
      train_parser.add_argument("--valid_batch_size"							,   type=SCREAMING_SNAKE_CASE							,   default=64							,   help="Batch size for validation."						)
      train_parser.add_argument("--learning_rate"							,   type=SCREAMING_SNAKE_CASE							,   default=3e-5							,   help="Learning rate."						)
      train_parser.add_argument("--adam_epsilon"							,   type=SCREAMING_SNAKE_CASE							,   default=1e-0_8							,   help="Epsilon for Adam optimizer."						)
      train_parser.set_defaults(func=SCREAMING_SNAKE_CASE						)







 def __init__(		self					:   Any							,   SCREAMING_SNAKE_CASE					:   Namespace						):


      '''simple docstring'''






      UpperCamelCase__			:   Any       				=  logging.get_logger("transformers-cli/training"						)

      UpperCamelCase__			:   Dict       				=  "tf" if is_tf_available() else "torch"

      os.makedirs(args.output							,   exist_ok=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   Optional[int]       				=  args.output

      UpperCamelCase__			:   List[str]       				=  args.column_label
      UpperCamelCase__			:   int       				=  args.column_text
      UpperCamelCase__			:   int       				=  args.column_id

      self.logger.info(F'Loading {args.task} pipeline for {args.model}'						)
      if args.task == "text_classification":
           UpperCamelCase__			:   Dict       				=  TextClassificationPipeline.from_pretrained(args.model						)
      elif args.task == "token_classification":
           raise NotImplementedError
      elif args.task == "question_answering":
           raise NotImplementedError

      self.logger.info(F'Loading dataset from {args.train_data}'						)
      UpperCamelCase__			:   int       				=  Processor.create_from_csv(
          args.train_data							,   column_label=args.column_label							,   column_text=args.column_text							,   column_id=args.column_id							,   skip_first_row=args.skip_first_row							,   )
      UpperCamelCase__			:   Optional[int]       				=  None
      if args.validation_data:
           self.logger.info(F'Loading validation dataset from {args.validation_data}'						)
           UpperCamelCase__			:   int       				=  Processor.create_from_csv(
               args.validation_data							,   column_label=args.column_label							,   column_text=args.column_text							,   column_id=args.column_id							,   skip_first_row=args.skip_first_row							,   )

      UpperCamelCase__			:   str       				=  args.validation_split
      UpperCamelCase__			:   List[Any]       				=  args.train_batch_size
      UpperCamelCase__			:   Any       				=  args.valid_batch_size
      UpperCamelCase__			:   Any       				=  args.learning_rate
      UpperCamelCase__			:   Tuple       				=  args.adam_epsilon







 def 					__lowercase					(		self					:   List[str]						):


      '''simple docstring'''






      if self.framework == "tf":
           return self.run_tf()
      return self.run_torch()







 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      raise NotImplementedError







 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      self.pipeline.fit(
          self.train_dataset							,   validation_data=self.valid_dataset							,   validation_split=self.validation_split							,   learning_rate=self.learning_rate							,   adam_epsilon=self.adam_epsilon							,   train_batch_size=self.train_batch_size							,   valid_batch_size=self.valid_batch_size							,   )

      # Save trained pipeline
      self.pipeline.save_pretrained(self.output						)







import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
 import faiss





@require_faiss
class 			__a  (    A__       ):







 def 					__lowercase					(		self					:   Any						):


      '''simple docstring'''






      UpperCamelCase__			:   List[str]       				=  tempfile.mkdtemp()
      UpperCamelCase__			:   Any       				=  8

      # DPR tok
      UpperCamelCase__			:   List[Any]       				=  [
          "[UNK]",
          "[CLS]",
          "[SEP]",
          "[PAD]",
          "[MASK]",
          "want",
          "##want",
          "##ed",
          "wa",
          "un",
          "runn",
          "##ing",
          ",",
          "low",
          "lowest",
      ]
      UpperCamelCase__			:   Any       				=  os.path.join(self.tmpdirname							,   "dpr_tokenizer"						)
      os.makedirs(SCREAMING_SNAKE_CASE							,   exist_ok=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   Optional[int]       				=  os.path.join(SCREAMING_SNAKE_CASE							,   DPR_VOCAB_FILES_NAMES["vocab_file"]						)
      with open(self.vocab_file							,   "w"							,   encoding="utf-8"						) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)

      # BART tok
      UpperCamelCase__			:   str       				=  [
          "l",
          "o",
          "w",
          "e",
          "r",
          "s",
          "t",
          "i",
          "d",
          "n",
          "\u0120",
          "\u0120l",
          "\u0120n",
          "\u0120lo",
          "\u0120low",
          "er",
          "\u0120lowest",
          "\u0120newer",
          "\u0120wider",
          "<unk>",
      ]
      UpperCamelCase__			:   int       				=  dict(zip(SCREAMING_SNAKE_CASE							,   range(len(SCREAMING_SNAKE_CASE						)						)						)						)
      UpperCamelCase__			:   Optional[Any]       				=  ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
      UpperCamelCase__			:   Union[str, Any]       				=  {"unk_token": "<unk>"}

      UpperCamelCase__			:   Dict       				=  os.path.join(self.tmpdirname							,   "bart_tokenizer"						)
      os.makedirs(SCREAMING_SNAKE_CASE							,   exist_ok=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   List[str]       				=  os.path.join(SCREAMING_SNAKE_CASE							,   BART_VOCAB_FILES_NAMES["vocab_file"]						)
      UpperCamelCase__			:   str       				=  os.path.join(SCREAMING_SNAKE_CASE							,   BART_VOCAB_FILES_NAMES["merges_file"]						)
      with open(self.vocab_file							,   "w"							,   encoding="utf-8"						) as fp:
           fp.write(json.dumps(SCREAMING_SNAKE_CASE						) + "\n"						)
      with open(self.merges_file							,   "w"							,   encoding="utf-8"						) as fp:
           fp.write("\n".join(SCREAMING_SNAKE_CASE						)						)







 def 					__lowercase					(		self					:   int						):


      '''simple docstring'''






      return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname							,   "dpr_tokenizer"						)						)







 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname							,   "dpr_tokenizer"						)						)







 def 					__lowercase					(		self					:   Optional[int]						):


      '''simple docstring'''






      return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname							,   "bart_tokenizer"						)						)







 def 					__lowercase					(		self					:   Optional[Any]						):


      '''simple docstring'''






      shutil.rmtree(self.tmpdirname						)







 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      UpperCamelCase__			:   Tuple       				=  Dataset.from_dict(
          {
              "id": ["0", "1"],
              "text": ["foo", "bar"],
              "title": ["Foo", "Bar"],
              "embeddings": [np.ones(self.retrieval_vector_size						), 2 * np.ones(self.retrieval_vector_size						)],
          }						)
      dataset.add_faiss_index("embeddings"							,   string_factory="Flat"							,   metric_type=faiss.METRIC_INNER_PRODUCT						)
      return dataset







 def 					__lowercase					(		self					:   Optional[int]						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  self.get_dummy_dataset()
      UpperCamelCase__			:   List[Any]       				=  RagConfig(
          retrieval_vector_size=self.retrieval_vector_size							,   question_encoder=DPRConfig().to_dict()							,   generator=BartConfig().to_dict()							,   )
      with patch("transformers.models.rag.retrieval_rag.load_dataset"						) as mock_load_dataset:
           UpperCamelCase__			:   str       				=  dataset
           UpperCamelCase__			:   Optional[int]       				=  RagRetriever(
               SCREAMING_SNAKE_CASE							,   question_encoder_tokenizer=self.get_dpr_tokenizer()							,   generator_tokenizer=self.get_bart_tokenizer()							,   )
      return retriever







 def 					__lowercase					(		self					:   int							,   SCREAMING_SNAKE_CASE					:   bool						):


      '''simple docstring'''






      UpperCamelCase__			:   Dict       				=  self.get_dummy_dataset()
      UpperCamelCase__			:   Tuple       				=  RagConfig(
          retrieval_vector_size=self.retrieval_vector_size							,   question_encoder=DPRConfig().to_dict()							,   generator=BartConfig().to_dict()							,   index_name="custom"							,   )
      if from_disk:
           UpperCamelCase__			:   Optional[int]       				=  os.path.join(self.tmpdirname							,   "dataset"						)
           UpperCamelCase__			:   List[str]       				=  os.path.join(self.tmpdirname							,   "index.faiss"						)
           dataset.get_index("embeddings"						).save(os.path.join(self.tmpdirname							,   "index.faiss"						)						)
           dataset.drop_index("embeddings"						)
           dataset.save_to_disk(os.path.join(self.tmpdirname							,   "dataset"						)						)
           del dataset
           UpperCamelCase__			:   str       				=  RagRetriever(
               SCREAMING_SNAKE_CASE							,   question_encoder_tokenizer=self.get_dpr_tokenizer()							,   generator_tokenizer=self.get_bart_tokenizer()							,   )
      else:
           UpperCamelCase__			:   List[Any]       				=  RagRetriever(
               SCREAMING_SNAKE_CASE							,   question_encoder_tokenizer=self.get_dpr_tokenizer()							,   generator_tokenizer=self.get_bart_tokenizer()							,   index=CustomHFIndex(config.retrieval_vector_size							,   SCREAMING_SNAKE_CASE						)							,   )
      return retriever







 def 					__lowercase					(		self					:   int						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  Dataset.from_dict(
          {
              "id": ["0", "1"],
              "text": ["foo", "bar"],
              "title": ["Foo", "Bar"],
              "embeddings": [np.ones(self.retrieval_vector_size + 1						), 2 * np.ones(self.retrieval_vector_size + 1						)],
          }						)
      dataset.add_faiss_index("embeddings"							,   string_factory="Flat"							,   metric_type=faiss.METRIC_INNER_PRODUCT						)

      UpperCamelCase__			:   List[str]       				=  os.path.join(self.tmpdirname							,   "hf_bert_base.hnswSQ8_correct_phi_128.c_index"						)
      dataset.save_faiss_index("embeddings"							,   index_file_name + ".index.dpr"						)
      pickle.dump(dataset["id"]							,   open(index_file_name + ".index_meta.dpr"							,   "wb"						)						)

      UpperCamelCase__			:   Optional[int]       				=  os.path.join(self.tmpdirname							,   "psgs_w100.tsv.pkl"						)
      UpperCamelCase__			:   Tuple       				=  {sample["id"]: [sample["text"], sample["title"]] for sample in dataset}
      pickle.dump(SCREAMING_SNAKE_CASE							,   open(SCREAMING_SNAKE_CASE							,   "wb"						)						)

      UpperCamelCase__			:   List[str]       				=  RagConfig(
          retrieval_vector_size=self.retrieval_vector_size							,   question_encoder=DPRConfig().to_dict()							,   generator=BartConfig().to_dict()							,   index_name="legacy"							,   index_path=self.tmpdirname							,   )
      UpperCamelCase__			:   List[str]       				=  RagRetriever(
          SCREAMING_SNAKE_CASE							,   question_encoder_tokenizer=self.get_dpr_tokenizer()							,   generator_tokenizer=self.get_bart_tokenizer()						)
      return retriever







 def 					__lowercase					(		self					:   Optional[Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   Union[str, Any]       				=  1
      UpperCamelCase__			:   Tuple       				=  self.get_dummy_canonical_hf_index_retriever()
      UpperCamelCase__			:   str       				=  np.array(
          [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
      UpperCamelCase__				,     UpperCamelCase__				,     UpperCamelCase__			:   List[str]       				=  retriever.retrieve(SCREAMING_SNAKE_CASE							,   n_docs=SCREAMING_SNAKE_CASE						)
      self.assertEqual(retrieved_doc_embeds.shape							,   (2, n_docs, self.retrieval_vector_size)						)
      self.assertEqual(len(SCREAMING_SNAKE_CASE						)							,   2						)
      self.assertEqual(sorted(doc_dicts[0]						)							,   ["embeddings", "id", "text", "title"]						)
      self.assertEqual(len(doc_dicts[0]["id"]						)							,   SCREAMING_SNAKE_CASE						)
      self.assertEqual(doc_dicts[0]["id"][0]							,   "1"						)  # max inner product is reached with second doc
      self.assertEqual(doc_dicts[1]["id"][0]							,   "0"						)  # max inner product is reached with first doc
      self.assertListEqual(doc_ids.tolist()							,   [[1], [0]]						)







 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[int]       				=  self.get_dummy_canonical_hf_index_retriever()
      with tempfile.TemporaryDirectory() as tmp_dirname:
           with patch("transformers.models.rag.retrieval_rag.load_dataset"						) as mock_load_dataset:
                UpperCamelCase__			:   Optional[int]       				=  self.get_dummy_dataset()
                retriever.save_pretrained(SCREAMING_SNAKE_CASE						)
                UpperCamelCase__			:   Any       				=  RagRetriever.from_pretrained(SCREAMING_SNAKE_CASE						)
                self.assertIsInstance(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)
                UpperCamelCase__			:   str       				=  np.array(
                    [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
                UpperCamelCase__			:   Any       				=  retriever.retrieve(SCREAMING_SNAKE_CASE							,   n_docs=1						)
                self.assertTrue(out is not None						)







 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  1
      UpperCamelCase__			:   Dict       				=  self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   Tuple       				=  np.array(
          [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
      UpperCamelCase__				,     UpperCamelCase__				,     UpperCamelCase__			:   List[str]       				=  retriever.retrieve(SCREAMING_SNAKE_CASE							,   n_docs=SCREAMING_SNAKE_CASE						)
      self.assertEqual(retrieved_doc_embeds.shape							,   (2, n_docs, self.retrieval_vector_size)						)
      self.assertEqual(len(SCREAMING_SNAKE_CASE						)							,   2						)
      self.assertEqual(sorted(doc_dicts[0]						)							,   ["embeddings", "id", "text", "title"]						)
      self.assertEqual(len(doc_dicts[0]["id"]						)							,   SCREAMING_SNAKE_CASE						)
      self.assertEqual(doc_dicts[0]["id"][0]							,   "1"						)  # max inner product is reached with second doc
      self.assertEqual(doc_dicts[1]["id"][0]							,   "0"						)  # max inner product is reached with first doc
      self.assertListEqual(doc_ids.tolist()							,   [[1], [0]]						)







 def 					__lowercase					(		self					:   str						):


      '''simple docstring'''






      UpperCamelCase__			:   Any       				=  self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE						)
      with tempfile.TemporaryDirectory() as tmp_dirname:
           retriever.save_pretrained(SCREAMING_SNAKE_CASE						)
           UpperCamelCase__			:   Tuple       				=  RagRetriever.from_pretrained(SCREAMING_SNAKE_CASE						)
           self.assertIsInstance(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)
           UpperCamelCase__			:   Optional[int]       				=  np.array(
               [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
           UpperCamelCase__			:   Any       				=  retriever.retrieve(SCREAMING_SNAKE_CASE							,   n_docs=1						)
           self.assertTrue(out is not None						)







 def 					__lowercase					(		self					:   List[str]						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  1
      UpperCamelCase__			:   List[str]       				=  self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   Dict       				=  np.array(
          [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
      UpperCamelCase__				,     UpperCamelCase__				,     UpperCamelCase__			:   List[str]       				=  retriever.retrieve(SCREAMING_SNAKE_CASE							,   n_docs=SCREAMING_SNAKE_CASE						)
      self.assertEqual(retrieved_doc_embeds.shape							,   (2, n_docs, self.retrieval_vector_size)						)
      self.assertEqual(len(SCREAMING_SNAKE_CASE						)							,   2						)
      self.assertEqual(sorted(doc_dicts[0]						)							,   ["embeddings", "id", "text", "title"]						)
      self.assertEqual(len(doc_dicts[0]["id"]						)							,   SCREAMING_SNAKE_CASE						)
      self.assertEqual(doc_dicts[0]["id"][0]							,   "1"						)  # max inner product is reached with second doc
      self.assertEqual(doc_dicts[1]["id"][0]							,   "0"						)  # max inner product is reached with first doc
      self.assertListEqual(doc_ids.tolist()							,   [[1], [0]]						)







 def 					__lowercase					(		self					:   Union[str, Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   List[str]       				=  self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE						)
      with tempfile.TemporaryDirectory() as tmp_dirname:
           retriever.save_pretrained(SCREAMING_SNAKE_CASE						)
           UpperCamelCase__			:   Optional[int]       				=  RagRetriever.from_pretrained(SCREAMING_SNAKE_CASE						)
           self.assertIsInstance(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)
           UpperCamelCase__			:   Dict       				=  np.array(
               [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
           UpperCamelCase__			:   Union[str, Any]       				=  retriever.retrieve(SCREAMING_SNAKE_CASE							,   n_docs=1						)
           self.assertTrue(out is not None						)







 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      UpperCamelCase__			:   List[str]       				=  1
      UpperCamelCase__			:   Any       				=  self.get_dummy_legacy_index_retriever()
      UpperCamelCase__			:   Optional[int]       				=  np.array(
          [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
      UpperCamelCase__				,     UpperCamelCase__				,     UpperCamelCase__			:   Tuple       				=  retriever.retrieve(SCREAMING_SNAKE_CASE							,   n_docs=SCREAMING_SNAKE_CASE						)
      self.assertEqual(retrieved_doc_embeds.shape							,   (2, n_docs, self.retrieval_vector_size)						)
      self.assertEqual(len(SCREAMING_SNAKE_CASE						)							,   2						)
      self.assertEqual(sorted(doc_dicts[0]						)							,   ["text", "title"]						)
      self.assertEqual(len(doc_dicts[0]["text"]						)							,   SCREAMING_SNAKE_CASE						)
      self.assertEqual(doc_dicts[0]["text"][0]							,   "bar"						)  # max inner product is reached with second doc
      self.assertEqual(doc_dicts[1]["text"][0]							,   "foo"						)  # max inner product is reached with first doc
      self.assertListEqual(doc_ids.tolist()							,   [[1], [0]]						)







 def 					__lowercase					(		self					:   List[Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   Union[str, Any]       				=  self.get_dummy_legacy_index_retriever()
      with tempfile.TemporaryDirectory() as tmp_dirname:
           retriever.save_pretrained(SCREAMING_SNAKE_CASE						)
           UpperCamelCase__			:   str       				=  RagRetriever.from_pretrained(SCREAMING_SNAKE_CASE						)
           self.assertIsInstance(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)
           UpperCamelCase__			:   int       				=  np.array(
               [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
           UpperCamelCase__			:   int       				=  retriever.retrieve(SCREAMING_SNAKE_CASE							,   n_docs=1						)
           self.assertTrue(out is not None						)







 @require_torch
 @require_tokenizers
 @require_sentencepiece
 def 					__lowercase					(		self					:   int						):


      '''simple docstring'''






      import torch

      UpperCamelCase__			:   Optional[Any]       				=  1
      UpperCamelCase__			:   Optional[int]       				=  self.get_dummy_canonical_hf_index_retriever()
      UpperCamelCase__			:   Optional[Any]       				=  [[5, 7], [10, 11]]
      UpperCamelCase__			:   List[str]       				=  np.array(
          [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
      UpperCamelCase__			:   int       				=  retriever(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE							,   prefix=retriever.config.generator.prefix							,   n_docs=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__				,     UpperCamelCase__				,     UpperCamelCase__			:   List[str]       				=  (
          out["context_input_ids"],
          out["context_attention_mask"],
          out["retrieved_doc_embeds"],
      )
      self.assertEqual(retrieved_doc_embeds.shape							,   (2, n_docs, self.retrieval_vector_size)						)
      self.assertIsInstance(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)
      self.assertIsInstance(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)
      self.assertIsInstance(SCREAMING_SNAKE_CASE							,   np.ndarray						)

      UpperCamelCase__			:   List[Any]       				=  retriever(
          SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE							,   prefix=retriever.config.generator.prefix							,   n_docs=SCREAMING_SNAKE_CASE							,   return_tensors="pt"							,   )
      UpperCamelCase__				,     UpperCamelCase__				,     UpperCamelCase__				,     UpperCamelCase__			:   List[Any]       				=  (  # noqa: F841
          out["context_input_ids"],
          out["context_attention_mask"],
          out["retrieved_doc_embeds"],
          out["doc_ids"],
      )
      self.assertEqual(retrieved_doc_embeds.shape							,   (2, n_docs, self.retrieval_vector_size)						)
      self.assertIsInstance(SCREAMING_SNAKE_CASE							,   torch.Tensor						)
      self.assertIsInstance(SCREAMING_SNAKE_CASE							,   torch.Tensor						)
      self.assertIsInstance(SCREAMING_SNAKE_CASE							,   torch.Tensor						)







 @require_torch
 @require_tokenizers
 @require_sentencepiece
 def 					__lowercase					(		self					:   Union[str, Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[int]       				=  self.get_dpr_ctx_encoder_tokenizer()
      UpperCamelCase__			:   Union[str, Any]       				=  1
      UpperCamelCase__			:   Tuple       				=  self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE						)
      retriever.set_ctx_encoder_tokenizer(SCREAMING_SNAKE_CASE						)

      UpperCamelCase__			:   Tuple       				=  [[5, 7], [10, 11]]
      UpperCamelCase__			:   List[str]       				=  np.array(
          [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]							,   dtype=np.floataa						)
      UpperCamelCase__			:   Optional[int]       				=  retriever(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE							,   prefix=retriever.config.generator.prefix							,   n_docs=SCREAMING_SNAKE_CASE						)

      self.assertEqual(
          len(SCREAMING_SNAKE_CASE						)							,   6						)  # check whether the retriever output consist of 6 attributes including tokenized docs
      self.assertEqual(
          all(k in out for k in ("tokenized_doc_ids", "tokenized_doc_attention_mask")						)							,   SCREAMING_SNAKE_CASE						)  # check for doc token related keys in dictionary.