



"""simple docstring"""







import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
     import faiss







@require_faiss
class       UpperCamelCase__   (	lowercase_  ):




      """simple docstring"""




      def      SCREAMING_SNAKE_CASE__ (      self				:      Tuple       ):
        lowerCAmelCase_   :				Optional[Any]								= tempfile.mkdtemp()
        lowerCAmelCase_   :				int								= 8

        # DPR tok
        lowerCAmelCase_   :				List[Any]								= [
            '[UNK]',
            '[CLS]',
            '[SEP]',
            '[PAD]',
            '[MASK]',
            'want',
            '##want',
            '##ed',
            'wa',
            'un',
            'runn',
            '##ing',
            ',',
            'low',
            'lowest',
        ]
        lowerCAmelCase_   :				int								= os.path.join(self.tmpdirname    ,			'dpr_tokenizer'       )
        os.makedirs(SCREAMING_SNAKE_CASE_    ,			exist_ok=SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				Union[str, Any]								= os.path.join(SCREAMING_SNAKE_CASE_    ,			DPR_VOCAB_FILES_NAMES['vocab_file']       )
        with open(self.vocab_file    ,			'w'    ,			encoding='utf-8'       ) as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )

        # BART tok
        lowerCAmelCase_   :				Optional[Any]								= [
            'l',
            'o',
            'w',
            'e',
            'r',
            's',
            't',
            'i',
            'd',
            'n',
            '\u0120',
            '\u0120l',
            '\u0120n',
            '\u0120lo',
            '\u0120low',
            'er',
            '\u0120lowest',
            '\u0120newer',
            '\u0120wider',
            '<unk>',
        ]
        lowerCAmelCase_   :				Optional[Any]								= dict(zip(SCREAMING_SNAKE_CASE_    ,			range(len(SCREAMING_SNAKE_CASE_       )       )       )       )
        lowerCAmelCase_   :				Dict								= ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
        lowerCAmelCase_   :				List[Any]								= {'unk_token': '<unk>'}

        lowerCAmelCase_   :				Tuple								= os.path.join(self.tmpdirname    ,			'bart_tokenizer'       )
        os.makedirs(SCREAMING_SNAKE_CASE_    ,			exist_ok=SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				Union[str, Any]								= os.path.join(SCREAMING_SNAKE_CASE_    ,			BART_VOCAB_FILES_NAMES['vocab_file']       )
        lowerCAmelCase_   :				Dict								= os.path.join(SCREAMING_SNAKE_CASE_    ,			BART_VOCAB_FILES_NAMES['merges_file']       )
        with open(self.vocab_file    ,			'w'    ,			encoding='utf-8'       ) as fp:
          fp.write(json.dumps(SCREAMING_SNAKE_CASE_       ) + '\n'       )
        with open(self.merges_file    ,			'w'    ,			encoding='utf-8'       ) as fp:
          fp.write('\n'.join(SCREAMING_SNAKE_CASE_       )       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname    ,			'dpr_tokenizer'       )       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname    ,			'dpr_tokenizer'       )       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname    ,			'bart_tokenizer'       )       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[int]       ):
        shutil.rmtree(self.tmpdirname       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Dict       ):
        lowerCAmelCase_   :				Union[str, Any]								= Dataset.from_dict(
            {
                'id': ['0', '1'],
                'text': ['foo', 'bar'],
                'title': ['Foo', 'Bar'],
                'embeddings': [np.ones(self.retrieval_vector_size       ), 2 * np.ones(self.retrieval_vector_size       )],
            }       )
        dataset.add_faiss_index('embeddings'    ,			string_factory='Flat'    ,			metric_type=faiss.METRIC_INNER_PRODUCT       )
        return dataset



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        lowerCAmelCase_   :				Optional[Any]								= self.get_dummy_dataset()
        lowerCAmelCase_   :				Optional[Any]								= RagConfig(
            retrieval_vector_size=self.retrieval_vector_size    ,			question_encoder=DPRConfig().to_dict()    ,			generator=BartConfig().to_dict()    ,			)
        with patch('transformers.models.rag.retrieval_rag.load_dataset'       ) as mock_load_dataset:
          lowerCAmelCase_   :				Dict								= dataset
          lowerCAmelCase_   :				Dict								= RagRetriever(
              SCREAMING_SNAKE_CASE_    ,			question_encoder_tokenizer=self.get_dpr_tokenizer()    ,			generator_tokenizer=self.get_bart_tokenizer()    ,			)
        return retriever



      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]    ,			SCREAMING_SNAKE_CASE_				:      bool       ):
        lowerCAmelCase_   :				List[str]								= self.get_dummy_dataset()
        lowerCAmelCase_   :				str								= RagConfig(
            retrieval_vector_size=self.retrieval_vector_size    ,			question_encoder=DPRConfig().to_dict()    ,			generator=BartConfig().to_dict()    ,			index_name='custom'    ,			)
        if from_disk:
          lowerCAmelCase_   :				str								= os.path.join(self.tmpdirname    ,			'dataset'       )
          lowerCAmelCase_   :				Any								= os.path.join(self.tmpdirname    ,			'index.faiss'       )
          dataset.get_index('embeddings'       ).save(os.path.join(self.tmpdirname    ,			'index.faiss'       )       )
          dataset.drop_index('embeddings'       )
          dataset.save_to_disk(os.path.join(self.tmpdirname    ,			'dataset'       )       )
          del dataset
          lowerCAmelCase_   :				Optional[Any]								= RagRetriever(
              SCREAMING_SNAKE_CASE_    ,			question_encoder_tokenizer=self.get_dpr_tokenizer()    ,			generator_tokenizer=self.get_bart_tokenizer()    ,			)
        else:
          lowerCAmelCase_   :				List[Any]								= RagRetriever(
              SCREAMING_SNAKE_CASE_    ,			question_encoder_tokenizer=self.get_dpr_tokenizer()    ,			generator_tokenizer=self.get_bart_tokenizer()    ,			index=CustomHFIndex(config.retrieval_vector_size    ,			SCREAMING_SNAKE_CASE_       )    ,			)
        return retriever



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[str]       ):
        lowerCAmelCase_   :				List[str]								= Dataset.from_dict(
            {
                'id': ['0', '1'],
                'text': ['foo', 'bar'],
                'title': ['Foo', 'Bar'],
                'embeddings': [np.ones(self.retrieval_vector_size + 1       ), 2 * np.ones(self.retrieval_vector_size + 1       )],
            }       )
        dataset.add_faiss_index('embeddings'    ,			string_factory='Flat'    ,			metric_type=faiss.METRIC_INNER_PRODUCT       )

        lowerCAmelCase_   :				Any								= os.path.join(self.tmpdirname    ,			'hf_bert_base.hnswSQ8_correct_phi_128.c_index'       )
        dataset.save_faiss_index('embeddings'    ,			index_file_name + '.index.dpr'       )
        pickle.dump(dataset['id']    ,			open(index_file_name + '.index_meta.dpr'    ,			'wb'       )       )

        lowerCAmelCase_   :				Union[str, Any]								= os.path.join(self.tmpdirname    ,			'psgs_w100.tsv.pkl'       )
        lowerCAmelCase_   :				Dict								= {sample['id']: [sample['text'], sample['title']] for sample in dataset}
        pickle.dump(SCREAMING_SNAKE_CASE_    ,			open(SCREAMING_SNAKE_CASE_    ,			'wb'       )       )

        lowerCAmelCase_   :				Tuple								= RagConfig(
            retrieval_vector_size=self.retrieval_vector_size    ,			question_encoder=DPRConfig().to_dict()    ,			generator=BartConfig().to_dict()    ,			index_name='legacy'    ,			index_path=self.tmpdirname    ,			)
        lowerCAmelCase_   :				int								= RagRetriever(
            SCREAMING_SNAKE_CASE_    ,			question_encoder_tokenizer=self.get_dpr_tokenizer()    ,			generator_tokenizer=self.get_bart_tokenizer()       )
        return retriever



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        lowerCAmelCase_   :				List[str]								= 1
        lowerCAmelCase_   :				List[Any]								= self.get_dummy_canonical_hf_index_retriever()
        lowerCAmelCase_   :				Dict								= np.array(
            [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
        lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_   :				Any								= retriever.retrieve(SCREAMING_SNAKE_CASE_    ,			n_docs=SCREAMING_SNAKE_CASE_       )
        self.assertEqual(retrieved_doc_embeds.shape    ,			(2, n_docs, self.retrieval_vector_size)       )
        self.assertEqual(len(SCREAMING_SNAKE_CASE_       )    ,			2       )
        self.assertEqual(sorted(doc_dicts[0]       )    ,			['embeddings', 'id', 'text', 'title']       )
        self.assertEqual(len(doc_dicts[0]['id']       )    ,			SCREAMING_SNAKE_CASE_       )
        self.assertEqual(doc_dicts[0]['id'][0]    ,			'1'       )  # max inner product is reached with second doc
        self.assertEqual(doc_dicts[1]['id'][0]    ,			'0'       )  # max inner product is reached with first doc
        self.assertListEqual(doc_ids.tolist()    ,			[[1], [0]]       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        lowerCAmelCase_   :				Optional[int]								= self.get_dummy_canonical_hf_index_retriever()
        with tempfile.TemporaryDirectory() as tmp_dirname:
          with patch('transformers.models.rag.retrieval_rag.load_dataset'       ) as mock_load_dataset:
            lowerCAmelCase_   :				List[Any]								= self.get_dummy_dataset()
            retriever.save_pretrained(SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Optional[int]								= RagRetriever.from_pretrained(SCREAMING_SNAKE_CASE_       )
            self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				List[Any]								= np.array(
                [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
            lowerCAmelCase_   :				Union[str, Any]								= retriever.retrieve(SCREAMING_SNAKE_CASE_    ,			n_docs=1       )
            self.assertTrue(out is not None       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Dict       ):
        lowerCAmelCase_   :				List[Any]								= 1
        lowerCAmelCase_   :				str								= self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				Dict								= np.array(
            [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
        lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_   :				Union[str, Any]								= retriever.retrieve(SCREAMING_SNAKE_CASE_    ,			n_docs=SCREAMING_SNAKE_CASE_       )
        self.assertEqual(retrieved_doc_embeds.shape    ,			(2, n_docs, self.retrieval_vector_size)       )
        self.assertEqual(len(SCREAMING_SNAKE_CASE_       )    ,			2       )
        self.assertEqual(sorted(doc_dicts[0]       )    ,			['embeddings', 'id', 'text', 'title']       )
        self.assertEqual(len(doc_dicts[0]['id']       )    ,			SCREAMING_SNAKE_CASE_       )
        self.assertEqual(doc_dicts[0]['id'][0]    ,			'1'       )  # max inner product is reached with second doc
        self.assertEqual(doc_dicts[1]['id'][0]    ,			'0'       )  # max inner product is reached with first doc
        self.assertListEqual(doc_ids.tolist()    ,			[[1], [0]]       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      str       ):
        lowerCAmelCase_   :				List[Any]								= self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE_       )
        with tempfile.TemporaryDirectory() as tmp_dirname:
          retriever.save_pretrained(SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				str								= RagRetriever.from_pretrained(SCREAMING_SNAKE_CASE_       )
          self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Tuple								= np.array(
              [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
          lowerCAmelCase_   :				Tuple								= retriever.retrieve(SCREAMING_SNAKE_CASE_    ,			n_docs=1       )
          self.assertTrue(out is not None       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        lowerCAmelCase_   :				Optional[int]								= 1
        lowerCAmelCase_   :				str								= self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				Optional[int]								= np.array(
            [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
        lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_   :				List[Any]								= retriever.retrieve(SCREAMING_SNAKE_CASE_    ,			n_docs=SCREAMING_SNAKE_CASE_       )
        self.assertEqual(retrieved_doc_embeds.shape    ,			(2, n_docs, self.retrieval_vector_size)       )
        self.assertEqual(len(SCREAMING_SNAKE_CASE_       )    ,			2       )
        self.assertEqual(sorted(doc_dicts[0]       )    ,			['embeddings', 'id', 'text', 'title']       )
        self.assertEqual(len(doc_dicts[0]['id']       )    ,			SCREAMING_SNAKE_CASE_       )
        self.assertEqual(doc_dicts[0]['id'][0]    ,			'1'       )  # max inner product is reached with second doc
        self.assertEqual(doc_dicts[1]['id'][0]    ,			'0'       )  # max inner product is reached with first doc
        self.assertListEqual(doc_ids.tolist()    ,			[[1], [0]]       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        lowerCAmelCase_   :				Any								= self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE_       )
        with tempfile.TemporaryDirectory() as tmp_dirname:
          retriever.save_pretrained(SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Optional[Any]								= RagRetriever.from_pretrained(SCREAMING_SNAKE_CASE_       )
          self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Dict								= np.array(
              [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
          lowerCAmelCase_   :				str								= retriever.retrieve(SCREAMING_SNAKE_CASE_    ,			n_docs=1       )
          self.assertTrue(out is not None       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Dict       ):
        lowerCAmelCase_   :				Dict								= 1
        lowerCAmelCase_   :				Union[str, Any]								= self.get_dummy_legacy_index_retriever()
        lowerCAmelCase_   :				List[Any]								= np.array(
            [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
        lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_   :				Union[str, Any]								= retriever.retrieve(SCREAMING_SNAKE_CASE_    ,			n_docs=SCREAMING_SNAKE_CASE_       )
        self.assertEqual(retrieved_doc_embeds.shape    ,			(2, n_docs, self.retrieval_vector_size)       )
        self.assertEqual(len(SCREAMING_SNAKE_CASE_       )    ,			2       )
        self.assertEqual(sorted(doc_dicts[0]       )    ,			['text', 'title']       )
        self.assertEqual(len(doc_dicts[0]['text']       )    ,			SCREAMING_SNAKE_CASE_       )
        self.assertEqual(doc_dicts[0]['text'][0]    ,			'bar'       )  # max inner product is reached with second doc
        self.assertEqual(doc_dicts[1]['text'][0]    ,			'foo'       )  # max inner product is reached with first doc
        self.assertListEqual(doc_ids.tolist()    ,			[[1], [0]]       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        lowerCAmelCase_   :				Optional[Any]								= self.get_dummy_legacy_index_retriever()
        with tempfile.TemporaryDirectory() as tmp_dirname:
          retriever.save_pretrained(SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				int								= RagRetriever.from_pretrained(SCREAMING_SNAKE_CASE_       )
          self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				int								= np.array(
              [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
          lowerCAmelCase_   :				Dict								= retriever.retrieve(SCREAMING_SNAKE_CASE_    ,			n_docs=1       )
          self.assertTrue(out is not None       )



      @require_torch
      @require_tokenizers
      @require_sentencepiece
      def      SCREAMING_SNAKE_CASE__ (      self				:      Tuple       ):
        import torch

        lowerCAmelCase_   :				Optional[Any]								= 1
        lowerCAmelCase_   :				Tuple								= self.get_dummy_canonical_hf_index_retriever()
        lowerCAmelCase_   :				Dict								= [[5, 7], [1_0, 1_1]]
        lowerCAmelCase_   :				Dict								= np.array(
            [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
        lowerCAmelCase_   :				Any								= retriever(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			prefix=retriever.config.generator.prefix    ,			n_docs=SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_   :				Any								= (
            out['context_input_ids'],
            out['context_attention_mask'],
            out['retrieved_doc_embeds'],
        )
        self.assertEqual(retrieved_doc_embeds.shape    ,			(2, n_docs, self.retrieval_vector_size)       )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			np.ndarray       )

        lowerCAmelCase_   :				List[str]								= retriever(
            SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			prefix=retriever.config.generator.prefix    ,			n_docs=SCREAMING_SNAKE_CASE_    ,			return_tensors='pt'    ,			)
        lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_   :				List[Any]								= (  # noqa: F841
            out['context_input_ids'],
            out['context_attention_mask'],
            out['retrieved_doc_embeds'],
            out['doc_ids'],
        )
        self.assertEqual(retrieved_doc_embeds.shape    ,			(2, n_docs, self.retrieval_vector_size)       )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			torch.Tensor       )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			torch.Tensor       )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_    ,			torch.Tensor       )





      @require_torch
      @require_tokenizers
      @require_sentencepiece
      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        lowerCAmelCase_   :				List[str]								= self.get_dpr_ctx_encoder_tokenizer()
        lowerCAmelCase_   :				Optional[Any]								= 1
        lowerCAmelCase_   :				str								= self.get_dummy_custom_hf_index_retriever(from_disk=SCREAMING_SNAKE_CASE_       )
        retriever.set_ctx_encoder_tokenizer(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Tuple								= [[5, 7], [1_0, 1_1]]
        lowerCAmelCase_   :				List[Any]								= np.array(
            [np.ones(self.retrieval_vector_size       ), -np.ones(self.retrieval_vector_size       )]    ,			dtype=np.floataa       )
        lowerCAmelCase_   :				Dict								= retriever(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			prefix=retriever.config.generator.prefix    ,			n_docs=SCREAMING_SNAKE_CASE_       )

        self.assertEqual(
            len(SCREAMING_SNAKE_CASE_       )    ,			6       )  # check whether the retriever output consist of 6 attributes including tokenized docs
        self.assertEqual(
            all(k in out for k in ('tokenized_doc_ids', 'tokenized_doc_attention_mask')       )    ,			SCREAMING_SNAKE_CASE_       )  # check for doc token related keys in dictionary.










"""simple docstring"""







import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__		:				Dict          =	get_tests_dir("""fixtures/test_sentencepiece.model""")







@require_sentencepiece
@require_tokenizers
class       UpperCamelCase__   (	lowercase_,					unittest.TestCase  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     XLMRobertaTokenizer
      _SCREAMING_SNAKE_CASE  				=     XLMRobertaTokenizerFast
      _SCREAMING_SNAKE_CASE  				=     True
      _SCREAMING_SNAKE_CASE  				=     True



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[str]       ):
        super().setUp()

        # We have a SentencePiece fixture for testing
        lowerCAmelCase_   :				Any								= XLMRobertaTokenizer(SCREAMING_SNAKE_CASE_    ,			keep_accents=SCREAMING_SNAKE_CASE_       )
        tokenizer.save_pretrained(self.tmpdirname       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        lowerCAmelCase_   :				Any								= '<pad>'
        lowerCAmelCase_   :				int								= 1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE_       )    ,			SCREAMING_SNAKE_CASE_       )
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE_       )    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        lowerCAmelCase_   :				int								= list(self.get_tokenizer().get_vocab().keys()       )

        self.assertEqual(vocab_keys[0]    ,			'<s>'       )
        self.assertEqual(vocab_keys[1]    ,			'<pad>'       )
        self.assertEqual(vocab_keys[-1]    ,			'<mask>'       )
        self.assertEqual(len(SCREAMING_SNAKE_CASE_       )    ,			1_0_0_2       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        self.assertEqual(self.get_tokenizer().vocab_size    ,			1_0_0_2       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        lowerCAmelCase_   :				int								= XLMRobertaTokenizer(SCREAMING_SNAKE_CASE_    ,			keep_accents=SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Optional[Any]								= tokenizer.tokenize('This is a test'       )
        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			['▁This', '▁is', '▁a', '▁t', 'est']       )

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_       )    ,			[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]    ,			)

        lowerCAmelCase_   :				int								= tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_    ,			[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                'é',
                '.',
            ]    ,			)
        lowerCAmelCase_   :				Any								= tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_    ,			[
                value + tokenizer.fairseq_offset
                for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
                #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
            ]    ,			)

        lowerCAmelCase_   :				Union[str, Any]								= tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_    ,			[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '<unk>',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                '<unk>',
                '.',
            ]    ,			)



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        if not self.test_slow_tokenizer:
          # as we don't have a slow version, we can't compare the outputs between slow and fast versions
          return

        lowerCAmelCase_   :				List[str]								= (self.rust_tokenizer_class, 'hf-internal-testing/tiny-xlm-roberta', {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
            lowerCAmelCase_   :				str								= self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Optional[int]								= self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )

            lowerCAmelCase_   :				str								= tempfile.mkdtemp()

            lowerCAmelCase_   :				int								= tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Optional[int]								= tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_       )

            # Checks it save with the same files + the tokenizer.json file for the fast one
            self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files       )       )
            lowerCAmelCase_   :				Optional[Any]								= tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f       )
            self.assertSequenceEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

            # Checks everything loads correctly in the same way
            lowerCAmelCase_   :				int								= tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Optional[int]								= tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_       )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )       )
              # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
              # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

            shutil.rmtree(SCREAMING_SNAKE_CASE_       )

            # Save tokenizer rust, legacy_format=True
            lowerCAmelCase_   :				str								= tempfile.mkdtemp()

            lowerCAmelCase_   :				List[str]								= tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_    ,			legacy_format=SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				str								= tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_       )

            # Checks it save with the same files
            self.assertSequenceEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

            # Checks everything loads correctly in the same way
            lowerCAmelCase_   :				str								= tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Union[str, Any]								= tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_       )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )       )

            shutil.rmtree(SCREAMING_SNAKE_CASE_       )

            # Save tokenizer rust, legacy_format=False
            lowerCAmelCase_   :				int								= tempfile.mkdtemp()

            lowerCAmelCase_   :				Optional[Any]								= tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_    ,			legacy_format=SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				int								= tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_       )

            # Checks it saved the tokenizer.json file
            self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files       )       )

            # Checks everything loads correctly in the same way
            lowerCAmelCase_   :				List[str]								= tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Optional[int]								= tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_       )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )       )

            shutil.rmtree(SCREAMING_SNAKE_CASE_       )



      @cached_property
      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        return XLMRobertaTokenizer.from_pretrained('xlm-roberta-base'       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        with tempfile.NamedTemporaryFile() as f:
          shutil.copyfile(SCREAMING_SNAKE_CASE_    ,			f.name       )
          lowerCAmelCase_   :				Tuple								= XLMRobertaTokenizer(f.name    ,			keep_accents=SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Dict								= pickle.dumps(SCREAMING_SNAKE_CASE_       )
        pickle.loads(SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        if not self.test_rust_tokenizer:
          return

        lowerCAmelCase_   :				Union[str, Any]								= self.get_tokenizer()
        lowerCAmelCase_   :				Dict								= self.get_rust_tokenizer()

        lowerCAmelCase_   :				Tuple								= 'I was born in 92000, and this is falsé.'

        lowerCAmelCase_   :				Union[str, Any]								= tokenizer.tokenize(SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				Dict								= rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Any								= tokenizer.encode(SCREAMING_SNAKE_CASE_    ,			add_special_tokens=SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				List[str]								= rust_tokenizer.encode(SCREAMING_SNAKE_CASE_    ,			add_special_tokens=SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				int								= self.get_rust_tokenizer()
        lowerCAmelCase_   :				Any								= tokenizer.encode(SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				List[Any]								= rust_tokenizer.encode(SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )



      @slow
      def      SCREAMING_SNAKE_CASE__ (      self				:      Any       ):
        lowerCAmelCase_   :				Any								= 'Hello World!'
        lowerCAmelCase_   :				Union[str, Any]								= [0, 3_5_3_7_8, 6_6_6_1, 3_8, 2]
        # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
        # xlmr.eval()
        # xlmr.encode(symbols)

        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			self.big_tokenizer.encode(SCREAMING_SNAKE_CASE_       )       )



      @slow
      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        lowerCAmelCase_   :				Tuple								= (
            'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
            ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
        )
        lowerCAmelCase_   :				int								= [
            0,
            3_2_9_3,
            8_3,
            1_0,
            4_5_5_2,
            4_9_8_9,
            7_9_8_6,
            6_7_8,
            1_0,
            5_9_1_5,
            1_1_1,
            1_7_9_4_5_9,
            1_2_4_8_5_0,
            4,
            6_0_4_4,
            2_3_7,
            1_2,
            6,
            5,
            6,
            4,
            6_7_8_0,
            7_0_5,
            1_5,
            1_3_8_8,
            4_4,
            3_7_8,
            1_0_1_1_4,
            7_1_1,
            1_5_2,
            2_0,
            6,
            5,
            2_2_3_7_6,
            6_4_2,
            1_2_2_1,
            1_5_1_9_0,
            3_4_1_5_3,
            4_5_0,
            5_6_0_8,
            9_5_9,
            1_1_1_9,
            5_7_7_0_2,
            1_3_6,
            1_8_6,
            4_7,
            1_0_9_8,
            2_9_3_6_7,
            4_7,
            # 4426, # What fairseq tokenizes from "<unk>": "_<"
            # 3678, # What fairseq tokenizes from "<unk>": "unk"
            # 2740, # What fairseq tokenizes from "<unk>": ">"
            3,  # What we tokenize from "<unk>": "<unk>"
            6,  # Residue from the tokenization: an extra sentencepiece underline
            4,
            6_0_4_4,
            2_3_7,
            6_2_8_4,
            5_0_9_0_1,
            5_2_8,
            3_1,
            9_0,
            3_4,
            9_2_7,
            2,
        ]
        # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
        # xlmr.eval()
        # xlmr.encode(symbols)

        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			self.big_tokenizer.encode(SCREAMING_SNAKE_CASE_       )       )





      @slow
      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        # fmt: off
        lowerCAmelCase_   :				List[str]								= {'input_ids': [[0, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [0, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=SCREAMING_SNAKE_CASE_    ,			model_name='xlm-roberta-base'    ,			revision='d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'    ,			)





