import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
   import faiss





@require_faiss
class 		_lowercase						(					snake_case_       ):

 def 						SCREAMING_SNAKE_CASE__   ( self	:							str			)   ->						Dict:

  """simple docstring"""





  UpperCamelCase_					:						Dict =					tempfile.mkdtemp()
  UpperCamelCase_					:						List[str] =					8

  # DPR tok
  UpperCamelCase_					:						List[str] =					[
      '[UNK]',
      '[CLS]',
      '[SEP]',
      '[PAD]',
      '[MASK]',
      'want',
      '##want',
      '##ed',
      'wa',
      'un',
      'runn',
      '##ing',
      ',',
      'low',
      'lowest',
  ]
  UpperCamelCase_					:						int =					os.path.join(self.tmpdirname   ,      'dpr_tokenizer'			)
  os.makedirs(snake_case   ,      exist_ok=snake_case			)
  UpperCamelCase_					:						Optional[Any] =					os.path.join(snake_case   ,      DPR_VOCAB_FILES_NAMES['vocab_file']			)
  with open(self.vocab_file   ,      'w'   ,      encoding='utf-8'			) as vocab_writer:
   vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]			)			)

  # BART tok
  UpperCamelCase_					:						Union[str, Any] =					[
      'l',
      'o',
      'w',
      'e',
      'r',
      's',
      't',
      'i',
      'd',
      'n',
      '\u0120',
      '\u0120l',
      '\u0120n',
      '\u0120lo',
      '\u0120low',
      'er',
      '\u0120lowest',
      '\u0120newer',
      '\u0120wider',
      '<unk>',
  ]
  UpperCamelCase_					:						Any =					dict(zip(snake_case   ,      range(len(snake_case			)			)			)			)
  UpperCamelCase_					:						int =					['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
  UpperCamelCase_					:						int =					{'unk_token': '<unk>'}

  UpperCamelCase_					:						Any =					os.path.join(self.tmpdirname   ,      'bart_tokenizer'			)
  os.makedirs(snake_case   ,      exist_ok=snake_case			)
  UpperCamelCase_					:						List[str] =					os.path.join(snake_case   ,      BART_VOCAB_FILES_NAMES['vocab_file']			)
  UpperCamelCase_					:						Dict =					os.path.join(snake_case   ,      BART_VOCAB_FILES_NAMES['merges_file']			)
  with open(self.vocab_file   ,      'w'   ,      encoding='utf-8'			) as fp:
   fp.write(json.dumps(snake_case			) + '\n'			)
  with open(self.merges_file   ,      'w'   ,      encoding='utf-8'			) as fp:
   fp.write('\n'.join(snake_case			)			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Union[str, Any]			)   ->						DPRQuestionEncoderTokenizer:

  """simple docstring"""





  return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,      'dpr_tokenizer'			)			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							List[str]			)   ->						DPRContextEncoderTokenizer:

  """simple docstring"""





  return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,      'dpr_tokenizer'			)			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Union[str, Any]			)   ->						BartTokenizer:

  """simple docstring"""





  return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,      'bart_tokenizer'			)			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							str			)   ->						Union[str, Any]:

  """simple docstring"""





  shutil.rmtree(self.tmpdirname			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Tuple			)   ->						Dict:

  """simple docstring"""





  UpperCamelCase_					:						Union[str, Any] =					Dataset.from_dict(
      {
          'id': ['0', '1'],
          'text': ['foo', 'bar'],
          'title': ['Foo', 'Bar'],
          'embeddings': [np.ones(self.retrieval_vector_size			), 2 * np.ones(self.retrieval_vector_size			)],
      }			)
  dataset.add_faiss_index('embeddings'   ,      string_factory='Flat'   ,      metric_type=faiss.METRIC_INNER_PRODUCT			)
  return dataset

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Union[str, Any]			)   ->						List[str]:

  """simple docstring"""





  UpperCamelCase_					:						Dict =					self.get_dummy_dataset()
  UpperCamelCase_					:						Optional[Any] =					RagConfig(
      retrieval_vector_size=self.retrieval_vector_size   ,      question_encoder=DPRConfig().to_dict()   ,      generator=BartConfig().to_dict()   ,      )
  with patch('transformers.models.rag.retrieval_rag.load_dataset'			) as mock_load_dataset:
   UpperCamelCase_					:						Union[str, Any] =					dataset
   UpperCamelCase_					:						Optional[Any] =					RagRetriever(
       snake_case   ,      question_encoder_tokenizer=self.get_dpr_tokenizer()   ,      generator_tokenizer=self.get_bart_tokenizer()   ,      )
  return retriever

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Tuple   ,      snake_case	:							bool			)   ->						Tuple:

  """simple docstring"""





  UpperCamelCase_					:						List[Any] =					self.get_dummy_dataset()
  UpperCamelCase_					:						Optional[int] =					RagConfig(
      retrieval_vector_size=self.retrieval_vector_size   ,      question_encoder=DPRConfig().to_dict()   ,      generator=BartConfig().to_dict()   ,      index_name='custom'   ,      )
  if from_disk:
   UpperCamelCase_					:						Union[str, Any] =					os.path.join(self.tmpdirname   ,      'dataset'			)
   UpperCamelCase_					:						Optional[int] =					os.path.join(self.tmpdirname   ,      'index.faiss'			)
   dataset.get_index('embeddings'			).save(os.path.join(self.tmpdirname   ,      'index.faiss'			)			)
   dataset.drop_index('embeddings'			)
   dataset.save_to_disk(os.path.join(self.tmpdirname   ,      'dataset'			)			)
   del dataset
   UpperCamelCase_					:						Optional[Any] =					RagRetriever(
       snake_case   ,      question_encoder_tokenizer=self.get_dpr_tokenizer()   ,      generator_tokenizer=self.get_bart_tokenizer()   ,      )
  else:
   UpperCamelCase_					:						Any =					RagRetriever(
       snake_case   ,      question_encoder_tokenizer=self.get_dpr_tokenizer()   ,      generator_tokenizer=self.get_bart_tokenizer()   ,      index=CustomHFIndex(config.retrieval_vector_size   ,      snake_case			)   ,      )
  return retriever

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Optional[Any]			)   ->						int:

  """simple docstring"""





  UpperCamelCase_					:						Union[str, Any] =					Dataset.from_dict(
      {
          'id': ['0', '1'],
          'text': ['foo', 'bar'],
          'title': ['Foo', 'Bar'],
          'embeddings': [np.ones(self.retrieval_vector_size + 1			), 2 * np.ones(self.retrieval_vector_size + 1			)],
      }			)
  dataset.add_faiss_index('embeddings'   ,      string_factory='Flat'   ,      metric_type=faiss.METRIC_INNER_PRODUCT			)

  UpperCamelCase_					:						Optional[int] =					os.path.join(self.tmpdirname   ,      'hf_bert_base.hnswSQ8_correct_phi_128.c_index'			)
  dataset.save_faiss_index('embeddings'   ,      index_file_name + '.index.dpr'			)
  pickle.dump(dataset['id']   ,      open(index_file_name + '.index_meta.dpr'   ,      'wb'			)			)

  UpperCamelCase_					:						Optional[int] =					os.path.join(self.tmpdirname   ,      'psgs_w100.tsv.pkl'			)
  UpperCamelCase_					:						int =					{sample['id']: [sample['text'], sample['title']] for sample in dataset}
  pickle.dump(snake_case   ,      open(snake_case   ,      'wb'			)			)

  UpperCamelCase_					:						Union[str, Any] =					RagConfig(
      retrieval_vector_size=self.retrieval_vector_size   ,      question_encoder=DPRConfig().to_dict()   ,      generator=BartConfig().to_dict()   ,      index_name='legacy'   ,      index_path=self.tmpdirname   ,      )
  UpperCamelCase_					:						List[Any] =					RagRetriever(
      snake_case   ,      question_encoder_tokenizer=self.get_dpr_tokenizer()   ,      generator_tokenizer=self.get_bart_tokenizer()			)
  return retriever

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Dict			)   ->						Tuple:

  """simple docstring"""





  UpperCamelCase_					:						Optional[Any] =					1
  UpperCamelCase_					:						str =					self.get_dummy_canonical_hf_index_retriever()
  UpperCamelCase_					:						List[Any] =					np.array(
      [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
  UpperCamelCase_, UpperCamelCase_, UpperCamelCase_					:						Any =					retriever.retrieve(snake_case   ,      n_docs=snake_case			)
  self.assertEqual(retrieved_doc_embeds.shape   ,      (2, n_docs, self.retrieval_vector_size)			)
  self.assertEqual(len(snake_case			)   ,      2			)
  self.assertEqual(sorted(doc_dicts[0]			)   ,      ['embeddings', 'id', 'text', 'title']			)
  self.assertEqual(len(doc_dicts[0]['id']			)   ,      snake_case			)
  self.assertEqual(doc_dicts[0]['id'][0]   ,      '1'			)  # max inner product is reached with second doc
  self.assertEqual(doc_dicts[1]['id'][0]   ,      '0'			)  # max inner product is reached with first doc
  self.assertListEqual(doc_ids.tolist()   ,      [[1], [0]]			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Any			)   ->						Dict:

  """simple docstring"""





  UpperCamelCase_					:						Optional[Any] =					self.get_dummy_canonical_hf_index_retriever()
  with tempfile.TemporaryDirectory() as tmp_dirname:
   with patch('transformers.models.rag.retrieval_rag.load_dataset'			) as mock_load_dataset:
    UpperCamelCase_					:						List[Any] =					self.get_dummy_dataset()
    retriever.save_pretrained(snake_case			)
    UpperCamelCase_					:						str =					RagRetriever.from_pretrained(snake_case			)
    self.assertIsInstance(snake_case   ,      snake_case			)
    UpperCamelCase_					:						Any =					np.array(
        [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
    UpperCamelCase_					:						List[str] =					retriever.retrieve(snake_case   ,      n_docs=1			)
    self.assertTrue(out is not None			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Optional[int]			)   ->						Optional[Any]:

  """simple docstring"""





  UpperCamelCase_					:						List[str] =					1
  UpperCamelCase_					:						Optional[int] =					self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
  UpperCamelCase_					:						List[str] =					np.array(
      [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
  UpperCamelCase_, UpperCamelCase_, UpperCamelCase_					:						Union[str, Any] =					retriever.retrieve(snake_case   ,      n_docs=snake_case			)
  self.assertEqual(retrieved_doc_embeds.shape   ,      (2, n_docs, self.retrieval_vector_size)			)
  self.assertEqual(len(snake_case			)   ,      2			)
  self.assertEqual(sorted(doc_dicts[0]			)   ,      ['embeddings', 'id', 'text', 'title']			)
  self.assertEqual(len(doc_dicts[0]['id']			)   ,      snake_case			)
  self.assertEqual(doc_dicts[0]['id'][0]   ,      '1'			)  # max inner product is reached with second doc
  self.assertEqual(doc_dicts[1]['id'][0]   ,      '0'			)  # max inner product is reached with first doc
  self.assertListEqual(doc_ids.tolist()   ,      [[1], [0]]			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							List[Any]			)   ->						Any:

  """simple docstring"""





  UpperCamelCase_					:						Any =					self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
  with tempfile.TemporaryDirectory() as tmp_dirname:
   retriever.save_pretrained(snake_case			)
   UpperCamelCase_					:						Union[str, Any] =					RagRetriever.from_pretrained(snake_case			)
   self.assertIsInstance(snake_case   ,      snake_case			)
   UpperCamelCase_					:						Dict =					np.array(
       [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
   UpperCamelCase_					:						List[str] =					retriever.retrieve(snake_case   ,      n_docs=1			)
   self.assertTrue(out is not None			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							List[Any]			)   ->						Union[str, Any]:

  """simple docstring"""





  UpperCamelCase_					:						List[str] =					1
  UpperCamelCase_					:						Optional[Any] =					self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
  UpperCamelCase_					:						Tuple =					np.array(
      [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
  UpperCamelCase_, UpperCamelCase_, UpperCamelCase_					:						Optional[int] =					retriever.retrieve(snake_case   ,      n_docs=snake_case			)
  self.assertEqual(retrieved_doc_embeds.shape   ,      (2, n_docs, self.retrieval_vector_size)			)
  self.assertEqual(len(snake_case			)   ,      2			)
  self.assertEqual(sorted(doc_dicts[0]			)   ,      ['embeddings', 'id', 'text', 'title']			)
  self.assertEqual(len(doc_dicts[0]['id']			)   ,      snake_case			)
  self.assertEqual(doc_dicts[0]['id'][0]   ,      '1'			)  # max inner product is reached with second doc
  self.assertEqual(doc_dicts[1]['id'][0]   ,      '0'			)  # max inner product is reached with first doc
  self.assertListEqual(doc_ids.tolist()   ,      [[1], [0]]			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							int			)   ->						List[str]:

  """simple docstring"""





  UpperCamelCase_					:						int =					self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
  with tempfile.TemporaryDirectory() as tmp_dirname:
   retriever.save_pretrained(snake_case			)
   UpperCamelCase_					:						Tuple =					RagRetriever.from_pretrained(snake_case			)
   self.assertIsInstance(snake_case   ,      snake_case			)
   UpperCamelCase_					:						Union[str, Any] =					np.array(
       [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
   UpperCamelCase_					:						Tuple =					retriever.retrieve(snake_case   ,      n_docs=1			)
   self.assertTrue(out is not None			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Optional[Any]			)   ->						Tuple:

  """simple docstring"""





  UpperCamelCase_					:						Union[str, Any] =					1
  UpperCamelCase_					:						List[Any] =					self.get_dummy_legacy_index_retriever()
  UpperCamelCase_					:						Optional[Any] =					np.array(
      [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
  UpperCamelCase_, UpperCamelCase_, UpperCamelCase_					:						List[str] =					retriever.retrieve(snake_case   ,      n_docs=snake_case			)
  self.assertEqual(retrieved_doc_embeds.shape   ,      (2, n_docs, self.retrieval_vector_size)			)
  self.assertEqual(len(snake_case			)   ,      2			)
  self.assertEqual(sorted(doc_dicts[0]			)   ,      ['text', 'title']			)
  self.assertEqual(len(doc_dicts[0]['text']			)   ,      snake_case			)
  self.assertEqual(doc_dicts[0]['text'][0]   ,      'bar'			)  # max inner product is reached with second doc
  self.assertEqual(doc_dicts[1]['text'][0]   ,      'foo'			)  # max inner product is reached with first doc
  self.assertListEqual(doc_ids.tolist()   ,      [[1], [0]]			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Any			)   ->						int:

  """simple docstring"""





  UpperCamelCase_					:						int =					self.get_dummy_legacy_index_retriever()
  with tempfile.TemporaryDirectory() as tmp_dirname:
   retriever.save_pretrained(snake_case			)
   UpperCamelCase_					:						Optional[Any] =					RagRetriever.from_pretrained(snake_case			)
   self.assertIsInstance(snake_case   ,      snake_case			)
   UpperCamelCase_					:						Tuple =					np.array(
       [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
   UpperCamelCase_					:						Any =					retriever.retrieve(snake_case   ,      n_docs=1			)
   self.assertTrue(out is not None			)

 @require_torch
 @require_tokenizers
 @require_sentencepiece
 def 						SCREAMING_SNAKE_CASE__   ( self	:							str			)   ->						Optional[Any]:

  """simple docstring"""





  import torch

  UpperCamelCase_					:						Dict =					1
  UpperCamelCase_					:						Optional[Any] =					self.get_dummy_canonical_hf_index_retriever()
  UpperCamelCase_					:						Tuple =					[[5, 7], [1_0, 1_1]]
  UpperCamelCase_					:						Union[str, Any] =					np.array(
      [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
  UpperCamelCase_					:						Union[str, Any] =					retriever(snake_case   ,      snake_case   ,      prefix=retriever.config.generator.prefix   ,      n_docs=snake_case			)
  UpperCamelCase_, UpperCamelCase_, UpperCamelCase_					:						str =					(
      out['context_input_ids'],
      out['context_attention_mask'],
      out['retrieved_doc_embeds'],
  )
  self.assertEqual(retrieved_doc_embeds.shape   ,      (2, n_docs, self.retrieval_vector_size)			)
  self.assertIsInstance(snake_case   ,      snake_case			)
  self.assertIsInstance(snake_case   ,      snake_case			)
  self.assertIsInstance(snake_case   ,      np.ndarray			)

  UpperCamelCase_					:						List[Any] =					retriever(
      snake_case   ,      snake_case   ,      prefix=retriever.config.generator.prefix   ,      n_docs=snake_case   ,      return_tensors='pt'   ,      )
  UpperCamelCase_, UpperCamelCase_, UpperCamelCase_, UpperCamelCase_					:						List[str] =					(  # noqa: F841
      out['context_input_ids'],
      out['context_attention_mask'],
      out['retrieved_doc_embeds'],
      out['doc_ids'],
  )
  self.assertEqual(retrieved_doc_embeds.shape   ,      (2, n_docs, self.retrieval_vector_size)			)
  self.assertIsInstance(snake_case   ,      torch.Tensor			)
  self.assertIsInstance(snake_case   ,      torch.Tensor			)
  self.assertIsInstance(snake_case   ,      torch.Tensor			)


 @require_torch
 @require_tokenizers
 @require_sentencepiece
 def 						SCREAMING_SNAKE_CASE__   ( self	:							Dict			)   ->						Union[str, Any]:

  """simple docstring"""





  UpperCamelCase_					:						Tuple =					self.get_dpr_ctx_encoder_tokenizer()
  UpperCamelCase_					:						Union[str, Any] =					1
  UpperCamelCase_					:						List[Any] =					self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
  retriever.set_ctx_encoder_tokenizer(snake_case			)

  UpperCamelCase_					:						Optional[int] =					[[5, 7], [1_0, 1_1]]
  UpperCamelCase_					:						Union[str, Any] =					np.array(
      [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]   ,      dtype=np.floataa			)
  UpperCamelCase_					:						Dict =					retriever(snake_case   ,      snake_case   ,      prefix=retriever.config.generator.prefix   ,      n_docs=snake_case			)

  self.assertEqual(
      len(snake_case			)   ,      6			)  # check whether the retriever output consist of 6 attributes including tokenized docs
  self.assertEqual(
      all(k in out for k in ('tokenized_doc_ids', 'tokenized_doc_attention_mask')			)   ,      snake_case			)  # check for doc token related keys in dictionary.




import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin





class 		_lowercase						(					snake_case_						,      unittest.TestCase       ):
 lowercase				     =   XLMTokenizer
 lowercase				     =   False

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Any			)   ->						List[Any]:

  """simple docstring"""





  super().setUp()

  # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
  UpperCamelCase_					:						List[Any] =					[
      'l',
      'o',
      'w',
      'e',
      'r',
      's',
      't',
      'i',
      'd',
      'n',
      'w</w>',
      'r</w>',
      't</w>',
      'lo',
      'low',
      'er</w>',
      'low</w>',
      'lowest</w>',
      'newer</w>',
      'wider</w>',
      '<unk>',
  ]
  UpperCamelCase_					:						str =					dict(zip(snake_case   ,      range(len(snake_case			)			)			)			)
  UpperCamelCase_					:						Tuple =					['l o 123', 'lo w 1456', 'e r</w> 1789', '']

  UpperCamelCase_					:						Tuple =					os.path.join(self.tmpdirname   ,      VOCAB_FILES_NAMES['vocab_file']			)
  UpperCamelCase_					:						Any =					os.path.join(self.tmpdirname   ,      VOCAB_FILES_NAMES['merges_file']			)
  with open(self.vocab_file   ,      'w'			) as fp:
   fp.write(json.dumps(snake_case			)			)
  with open(self.merges_file   ,      'w'			) as fp:
   fp.write('\n'.join(snake_case			)			)

 def 						SCREAMING_SNAKE_CASE__   ( self	:							List[Any]   ,      snake_case	:							Union[str, Any]			)   ->						int:

  """simple docstring"""





  UpperCamelCase_					:						Tuple =					'lower newer'
  UpperCamelCase_					:						Optional[int] =					'lower newer'
  return input_text, output_text

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Union[str, Any]			)   ->						Tuple:

  """simple docstring"""





  UpperCamelCase_					:						int =					XLMTokenizer(self.vocab_file   ,      self.merges_file			)

  UpperCamelCase_					:						List[str] =					'lower'
  UpperCamelCase_					:						Optional[int] =					['low', 'er</w>']
  UpperCamelCase_					:						Optional[Any] =					tokenizer.tokenize(snake_case			)
  self.assertListEqual(snake_case   ,      snake_case			)

  UpperCamelCase_					:						List[Any] =					tokens + ['<unk>']
  UpperCamelCase_					:						int =					[1_4, 1_5, 2_0]
  self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case			)   ,      snake_case			)


 @slow
 def 						SCREAMING_SNAKE_CASE__   ( self	:							List[Any]			)   ->						Optional[int]:

  """simple docstring"""





  UpperCamelCase_					:						int =					XLMTokenizer.from_pretrained('xlm-mlm-en-2048'			)

  UpperCamelCase_					:						int =					tokenizer.encode('sequence builders'   ,      add_special_tokens=snake_case			)
  UpperCamelCase_					:						Optional[int] =					tokenizer.encode('multi-sequence build'   ,      add_special_tokens=snake_case			)

  UpperCamelCase_					:						Tuple =					tokenizer.build_inputs_with_special_tokens(snake_case			)
  UpperCamelCase_					:						Any =					tokenizer.build_inputs_with_special_tokens(snake_case   ,      snake_case			)

  assert encoded_sentence == [0] + text + [1]
  assert encoded_pair == [0] + text + [1] + text_a + [1]



