



import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation

UpperCamelCase__			=    logging.get_logger(__name__)

UpperCamelCase__			=    {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

UpperCamelCase__			=    {
    "vocab_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"
        ),
    },
}

UpperCamelCase__			=    {
    "microsoft/deberta-base": 5_12,
    "microsoft/deberta-large": 5_12,
    "microsoft/deberta-xlarge": 5_12,
    "microsoft/deberta-base-mnli": 5_12,
    "microsoft/deberta-large-mnli": 5_12,
    "microsoft/deberta-xlarge-mnli": 5_12,
}

UpperCamelCase__			=    {
    "microsoft/deberta-base": {"do_lower_case": False},
    "microsoft/deberta-large": {"do_lower_case": False},
}






def 			UpperCAmelCase__     (     )			->   Any:
						__lowercase											=					(
						    list(range(ord("""!""" )    ,							ord("""~""" ) + 1 ) ) + list(range(ord("""¡""" )    ,							ord("""¬""" ) + 1 ) ) + list(range(ord("""®""" )    ,							ord("""ÿ""" ) + 1 ) )
						)
						__lowercase											=					bs[:]
						__lowercase											=					0
						for b in range(2**8 ):
												if b not in bs:
																		bs.append(lowercase__ )
																		cs.append(2**8 + n )
																		n += 1
						__lowercase											=					[chr(lowercase__ ) for n in cs]
						return dict(zip(lowercase__    ,							lowercase__ ) )






def 			UpperCAmelCase__     (     lowercase__ )			->   Any:
						__lowercase											=					set()
						__lowercase											=					word[0]
						for char in word[1:]:
												pairs.add((prev_char, char) )
												__lowercase											=					char
						return pairs







class       _lowerCAmelCase (			_UpperCAmelCase      ):






			"""simple docstring"""





			lowercase__   :	Union[str, Any]  			=      VOCAB_FILES_NAMES
			lowercase__   :	List[str]  			=      PRETRAINED_VOCAB_FILES_MAP
			lowercase__   :	Dict  			=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase__   :	Dict  			=      ["""input_ids""", """attention_mask""", """token_type_ids"""]





			def __init__(					self						:							str      ,  lowercase						:							Union[str, Any]      ,  lowercase						:							Any      ,  lowercase						:							Optional[int]="replace"      ,  lowercase						:							Dict="[CLS]"      ,  lowercase						:							Optional[int]="[SEP]"      ,  lowercase						:							List[str]="[SEP]"      ,  lowercase						:							List[Any]="[CLS]"      ,  lowercase						:							Tuple="[UNK]"      ,  lowercase						:							Any="[PAD]"      ,  lowercase						:							Optional[Any]="[MASK]"      ,  lowercase						:							Any=False      ,  lowercase						:							Tuple=False      ,  **lowercase						:							Optional[int]      ,  )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					AddedToken(lowercase      ,  lstrip=lowercase      ,  rstrip=lowercase ) if isinstance(lowercase      ,  lowercase ) else bos_token
									__lowercase											=					AddedToken(lowercase      ,  lstrip=lowercase      ,  rstrip=lowercase ) if isinstance(lowercase      ,  lowercase ) else eos_token
									__lowercase											=					AddedToken(lowercase      ,  lstrip=lowercase      ,  rstrip=lowercase ) if isinstance(lowercase      ,  lowercase ) else sep_token
									__lowercase											=					AddedToken(lowercase      ,  lstrip=lowercase      ,  rstrip=lowercase ) if isinstance(lowercase      ,  lowercase ) else cls_token
									__lowercase											=					AddedToken(lowercase      ,  lstrip=lowercase      ,  rstrip=lowercase ) if isinstance(lowercase      ,  lowercase ) else unk_token
									__lowercase											=					AddedToken(lowercase      ,  lstrip=lowercase      ,  rstrip=lowercase ) if isinstance(lowercase      ,  lowercase ) else pad_token

									# Mask token behave like a normal word, i.e. include the space before it
									__lowercase											=					AddedToken(lowercase      ,  lstrip=lowercase      ,  rstrip=lowercase ) if isinstance(lowercase      ,  lowercase ) else mask_token

									super().__init__(
									    errors=lowercase      ,  bos_token=lowercase      ,  eos_token=lowercase      ,  unk_token=lowercase      ,  sep_token=lowercase      ,  cls_token=lowercase      ,  pad_token=lowercase      ,  mask_token=lowercase      ,  add_prefix_space=lowercase      ,  add_bos_token=lowercase      ,  **lowercase      ,  )
									__lowercase											=					add_bos_token

									with open(lowercase      ,  encoding="""utf-8""" ) as vocab_handle:
															__lowercase											=					json.load(lowercase )
									__lowercase											=					{v: k for k, v in self.encoder.items()}
									__lowercase											=					errors  # how to handle errors in decoding
									__lowercase											=					bytes_to_unicode()
									__lowercase											=					{v: k for k, v in self.byte_encoder.items()}
									with open(lowercase      ,  encoding="""utf-8""" ) as merges_handle:
															__lowercase											=					merges_handle.read().split("""\n""" )[1:-1]
									__lowercase											=					[tuple(merge.split() ) for merge in bpe_merges]
									__lowercase											=					dict(zip(lowercase      ,  range(len(lowercase ) ) ) )
									__lowercase											=					{}
									__lowercase											=					add_prefix_space

									# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
									__lowercase											=					re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" )





			@property
			# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
			def 		snake_case__      (					self						:							Union[str, Any] )    ->       Optional[int]:
									"""simple docstring"""







									return len(self.encoder )





			def 		snake_case__      (					self						:							Optional[Any] )    ->       List[Any]:
									"""simple docstring"""







									return dict(self.encoder      ,  **self.added_tokens_encoder )





			def 		snake_case__      (					self						:							Any      ,  lowercase						:							Optional[Any] )    ->       str:
									"""simple docstring"""







									if token in self.cache:
															return self.cache[token]
									__lowercase											=					tuple(lowercase )
									__lowercase											=					get_pairs(lowercase )

									if not pairs:
															return token

									while True:
															__lowercase											=					min(lowercase      ,  key=lambda lowercase : self.bpe_ranks.get(lowercase      ,  float("""inf""" ) ) )
															if bigram not in self.bpe_ranks:
																					break
															__lowercase						,      __lowercase											=					bigram
															__lowercase											=					[]
															__lowercase											=					0
															while i < len(lowercase ):
																					try:
																											__lowercase											=					word.index(lowercase      ,  lowercase )
																					except ValueError:
																											new_word.extend(word[i:] )
																											break
																					else:
																											new_word.extend(word[i:j] )
																											__lowercase											=					j

																					if word[i] == first and i < len(lowercase ) - 1 and word[i + 1] == second:
																											new_word.append(first + second )
																											i += 2
																					else:
																											new_word.append(word[i] )
																											i += 1
															__lowercase											=					tuple(lowercase )
															__lowercase											=					new_word
															if len(lowercase ) == 1:
																					break
															else:
																					__lowercase											=					get_pairs(lowercase )
									__lowercase											=					""" """.join(lowercase )
									__lowercase											=					word
									return word





			def 		snake_case__      (					self						:							Any      ,  lowercase						:							List[int]      ,  lowercase						:							Optional[List[int]] = None )    ->       List[int]:
									"""simple docstring"""







									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									__lowercase											=					[self.cls_token_id]
									__lowercase											=					[self.sep_token_id]
									return cls + token_ids_a + sep + token_ids_a + sep





			def 		snake_case__      (					self						:							Optional[Any]      ,  lowercase						:							List[int]      ,  lowercase						:							Optional[List[int]] = None      ,  lowercase						:							bool = False )    ->       List[int]:
									"""simple docstring"""







									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowercase      ,  token_ids_a=lowercase      ,  already_has_special_tokens=lowercase )

									if token_ids_a is None:
															return [1] + ([0] * len(lowercase )) + [1]
									return [1] + ([0] * len(lowercase )) + [1] + ([0] * len(lowercase )) + [1]





			def 		snake_case__      (					self						:							Optional[int]      ,  lowercase						:							List[int]      ,  lowercase						:							Optional[List[int]] = None )    ->       List[int]:
									"""simple docstring"""







									__lowercase											=					[self.sep_token_id]
									__lowercase											=					[self.cls_token_id]

									if token_ids_a is None:
															return len(cls + token_ids_a + sep ) * [0]
									return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





			def 		snake_case__      (					self						:							str      ,  lowercase						:							Any )    ->       Dict:
									"""simple docstring"""







									__lowercase											=					[]
									for token in re.findall(self.pat      ,  lowercase ):
															__lowercase											=					"""""".join(
															    self.byte_encoder[b] for b in token.encode("""utf-8""" ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase ).split(""" """ ) )
									return bpe_tokens





			def 		snake_case__      (					self						:							Any      ,  lowercase						:							Optional[Any] )    ->       List[str]:
									"""simple docstring"""







									return self.encoder.get(lowercase      ,  self.encoder.get(self.unk_token ) )





			def 		snake_case__      (					self						:							Tuple      ,  lowercase						:							Any )    ->       List[Any]:
									"""simple docstring"""







									return self.decoder.get(lowercase )





			def 		snake_case__      (					self						:							Dict      ,  lowercase						:							Tuple )    ->       List[str]:
									"""simple docstring"""







									__lowercase											=					"""""".join(lowercase )
									__lowercase											=					bytearray([self.byte_decoder[c] for c in text] ).decode("""utf-8"""      ,  errors=self.errors )
									return text





			def 		snake_case__      (					self						:							Optional[int]      ,  lowercase						:							str      ,  lowercase						:							Optional[str] = None )    ->       Tuple[str]:
									"""simple docstring"""







									if not os.path.isdir(lowercase ):
															logger.error(F"Vocabulary path ({save_directory}) should be a directory" )
															return
									__lowercase											=					os.path.join(
									    lowercase      ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )
									__lowercase											=					os.path.join(
									    lowercase      ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""] )

									with open(lowercase      ,  """w"""      ,  encoding="""utf-8""" ) as f:
															f.write(json.dumps(self.encoder      ,  indent=2      ,  sort_keys=lowercase      ,  ensure_ascii=lowercase ) + """\n""" )

									__lowercase											=					0
									with open(lowercase      ,  """w"""      ,  encoding="""utf-8""" ) as writer:
															writer.write("""#version: 0.2\n""" )
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,  key=lambda lowercase : kv[1] ):
																					if index != token_index:
																											logger.warning(
																											    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																											    """ Please check that the tokenizer is not corrupted!""" )
																											__lowercase											=					token_index
																					writer.write(""" """.join(lowercase ) + """\n""" )
																					index += 1

									return vocab_file, merge_file





			def 		snake_case__      (					self						:							Optional[int]      ,  lowercase						:							str      ,  lowercase						:							Any=False      ,  **lowercase						:							List[Any] )    ->       Dict:
									"""simple docstring"""







									__lowercase											=					kwargs.pop("""add_prefix_space"""      ,  self.add_prefix_space )
									if (is_split_into_words or add_prefix_space) and (len(lowercase ) > 0 and not text[0].isspace()):
															__lowercase											=					""" """ + text
									return (text, kwargs)







			def 		snake_case__      (					self						:							List[Any]      ,  lowercase						:							"Conversation" )    ->       List[int]:
									"""simple docstring"""







									__lowercase											=					[]
									for is_user, text in conversation.iter_texts():
															input_ids.extend(self.encode(lowercase      ,  add_special_tokens=lowercase ) + [self.eos_token_id] )
									if len(lowercase ) > self.model_max_length:
															__lowercase											=					input_ids[-self.model_max_length :]
									return input_ids










from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
			    TFXLMForMultipleChoice,
			    TFXLMForQuestionAnsweringSimple,
			    TFXLMForSequenceClassification,
			    TFXLMForTokenClassification,
			    TFXLMModel,
			    TFXLMWithLMHeadModel,
			    XLMConfig,
			)
class       _lowerCAmelCase :






			"""simple docstring"""





			def __init__(					self						:							Tuple      ,  lowercase						:							str      ,  )    ->       str:
									"""simple docstring"""







									__lowercase											=					parent
									__lowercase											=					13
									__lowercase											=					7
									__lowercase											=					True
									__lowercase											=					True
									__lowercase											=					True
									__lowercase											=					True
									__lowercase											=					True
									__lowercase											=					False
									__lowercase											=					False
									__lowercase											=					False
									__lowercase											=					2
									__lowercase											=					99
									__lowercase											=					0
									__lowercase											=					32
									__lowercase											=					2
									__lowercase											=					4
									__lowercase											=					0.1
									__lowercase											=					0.1
									__lowercase											=					512
									__lowercase											=					16
									__lowercase											=					2
									__lowercase											=					0.02
									__lowercase											=					3
									__lowercase											=					4
									__lowercase											=					"""last"""
									__lowercase											=					True
									__lowercase											=					None
									__lowercase											=					0





			def 		snake_case__      (					self						:							Dict )    ->       Tuple:
									"""simple docstring"""







									__lowercase											=					ids_tensor([self.batch_size, self.seq_length]      ,  self.vocab_size )
									__lowercase											=					random_attention_mask([self.batch_size, self.seq_length]      ,  dtype=tf.floataa )

									__lowercase											=					None
									if self.use_input_lengths:
															__lowercase											=					(
															    ids_tensor([self.batch_size]      ,  vocab_size=2 ) + self.seq_length - 2
															)  # small variation of seq_length

									__lowercase											=					None
									if self.use_token_type_ids:
															__lowercase											=					ids_tensor([self.batch_size, self.seq_length]      ,  self.n_langs )

									__lowercase											=					None
									__lowercase											=					None
									__lowercase											=					None
									if self.use_labels:
															__lowercase											=					ids_tensor([self.batch_size]      ,  self.type_sequence_label_size )
															__lowercase											=					ids_tensor([self.batch_size, self.seq_length]      ,  self.num_labels )
															__lowercase											=					ids_tensor([self.batch_size]      ,  2      ,  dtype=tf.floataa )
															__lowercase											=					ids_tensor([self.batch_size]      ,  self.num_choices )

									__lowercase											=					XLMConfig(
									    vocab_size=self.vocab_size      ,  n_special=self.n_special      ,  emb_dim=self.hidden_size      ,  n_layers=self.num_hidden_layers      ,  n_heads=self.num_attention_heads      ,  dropout=self.hidden_dropout_prob      ,  attention_dropout=self.attention_probs_dropout_prob      ,  gelu_activation=self.gelu_activation      ,  sinusoidal_embeddings=self.sinusoidal_embeddings      ,  asm=self.asm      ,  causal=self.causal      ,  n_langs=self.n_langs      ,  max_position_embeddings=self.max_position_embeddings      ,  initializer_range=self.initializer_range      ,  summary_type=self.summary_type      ,  use_proj=self.use_proj      ,  bos_token_id=self.bos_token_id      ,  )

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_lengths,
									    sequence_labels,
									    token_labels,
									    is_impossible_labels,
									    choice_labels,
									    input_mask,
									)





			def 		snake_case__      (					self						:							List[str]      ,  lowercase						:							Tuple      ,  lowercase						:							List[Any]      ,  lowercase						:							Optional[int]      ,  lowercase						:							Optional[int]      ,  lowercase						:							Optional[Any]      ,  lowercase						:							Optional[int]      ,  lowercase						:							Optional[Any]      ,  lowercase						:							int      ,  lowercase						:							List[str]      ,  )    ->       Dict:
									"""simple docstring"""







									__lowercase											=					TFXLMModel(config=lowercase )
									__lowercase											=					{"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
									__lowercase											=					model(lowercase )

									__lowercase											=					[input_ids, input_mask]
									__lowercase											=					model(lowercase )
									self.parent.assertEqual(result.last_hidden_state.shape      ,  (self.batch_size, self.seq_length, self.hidden_size) )





			def 		snake_case__      (					self						:							Dict      ,  lowercase						:							List[str]      ,  lowercase						:							Optional[int]      ,  lowercase						:							Optional[Any]      ,  lowercase						:							List[Any]      ,  lowercase						:							Optional[Any]      ,  lowercase						:							Dict      ,  lowercase						:							Dict      ,  lowercase						:							int      ,  lowercase						:							Union[str, Any]      ,  )    ->       List[Any]:
									"""simple docstring"""







									__lowercase											=					TFXLMWithLMHeadModel(lowercase )

									__lowercase											=					{"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
									__lowercase											=					model(lowercase )

									__lowercase											=					outputs

									self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.seq_length, self.vocab_size) )





			def 		snake_case__      (					self						:							Optional[Any]      ,  lowercase						:							Optional[Any]      ,  lowercase						:							Any      ,  lowercase						:							int      ,  lowercase						:							List[Any]      ,  lowercase						:							Any      ,  lowercase						:							int      ,  lowercase						:							List[Any]      ,  lowercase						:							List[str]      ,  lowercase						:							Optional[Any]      ,  )    ->       Dict:
									"""simple docstring"""







									__lowercase											=					TFXLMForQuestionAnsweringSimple(lowercase )

									__lowercase											=					{"""input_ids""": input_ids, """lengths""": input_lengths}

									__lowercase											=					model(lowercase )

									self.parent.assertEqual(result.start_logits.shape      ,  (self.batch_size, self.seq_length) )
									self.parent.assertEqual(result.end_logits.shape      ,  (self.batch_size, self.seq_length) )





			def 		snake_case__      (					self						:							Optional[int]      ,  lowercase						:							Tuple      ,  lowercase						:							List[str]      ,  lowercase						:							List[Any]      ,  lowercase						:							Optional[int]      ,  lowercase						:							Optional[int]      ,  lowercase						:							Any      ,  lowercase						:							Tuple      ,  lowercase						:							Any      ,  lowercase						:							Dict      ,  )    ->       List[Any]:
									"""simple docstring"""







									__lowercase											=					TFXLMForSequenceClassification(lowercase )

									__lowercase											=					{"""input_ids""": input_ids, """lengths""": input_lengths}

									__lowercase											=					model(lowercase )

									self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.type_sequence_label_size) )





			def 		snake_case__      (					self						:							Union[str, Any]      ,  lowercase						:							List[str]      ,  lowercase						:							Optional[int]      ,  lowercase						:							List[Any]      ,  lowercase						:							Optional[int]      ,  lowercase						:							Tuple      ,  lowercase						:							Optional[Any]      ,  lowercase						:							Tuple      ,  lowercase						:							List[Any]      ,  lowercase						:							Optional[Any]      ,  )    ->       int:
									"""simple docstring"""







									__lowercase											=					self.num_labels
									__lowercase											=					TFXLMForTokenClassification(config=lowercase )
									__lowercase											=					{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
									__lowercase											=					model(lowercase )
									self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.seq_length, self.num_labels) )





			def 		snake_case__      (					self						:							int      ,  lowercase						:							Tuple      ,  lowercase						:							Optional[Any]      ,  lowercase						:							str      ,  lowercase						:							str      ,  lowercase						:							Dict      ,  lowercase						:							Optional[Any]      ,  lowercase						:							Optional[Any]      ,  lowercase						:							List[str]      ,  lowercase						:							List[str]      ,  )    ->       List[str]:
									"""simple docstring"""







									__lowercase											=					self.num_choices
									__lowercase											=					TFXLMForMultipleChoice(config=lowercase )
									__lowercase											=					tf.tile(tf.expand_dims(lowercase      ,  1 )      ,  (1, self.num_choices, 1) )
									__lowercase											=					tf.tile(tf.expand_dims(lowercase      ,  1 )      ,  (1, self.num_choices, 1) )
									__lowercase											=					tf.tile(tf.expand_dims(lowercase      ,  1 )      ,  (1, self.num_choices, 1) )
									__lowercase											=					{
									    """input_ids""": multiple_choice_inputs_ids,
									    """attention_mask""": multiple_choice_input_mask,
									    """token_type_ids""": multiple_choice_token_type_ids,
									}
									__lowercase											=					model(lowercase )
									self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.num_choices) )







			def 		snake_case__      (					self						:							str )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					self.prepare_config_and_inputs()
									(
									    (
									    __lowercase
									)						,      (
									    __lowercase
									)						,      (
									    __lowercase
									)						,      (
									    __lowercase
									)						,      (
									    __lowercase
									)						,      (
									    __lowercase
									)						,      (
									    __lowercase
									)						,      (
									    __lowercase
									)						,      (
									    __lowercase
									)						,      
									)											=					config_and_inputs
									__lowercase											=					{
									    """input_ids""": input_ids,
									    """token_type_ids""": token_type_ids,
									    """langs""": token_type_ids,
									    """lengths""": input_lengths,
									}
									return config, inputs_dict





@require_tf
class       _lowerCAmelCase (			_UpperCAmelCase     ,       _UpperCAmelCase     ,       unittest.TestCase      ):






			"""simple docstring"""





			lowercase__   :	Tuple  			=      (
			    (
			        TFXLMModel,
			        TFXLMWithLMHeadModel,
			        TFXLMForSequenceClassification,
			        TFXLMForQuestionAnsweringSimple,
			        TFXLMForTokenClassification,
			        TFXLMForMultipleChoice,
			    )
			    if is_tf_available()
			    else ()
			)
			lowercase__   :	List[str]  			=      (
			    (TFXLMWithLMHeadModel,) if is_tf_available() else ()
			)  # TODO (PVP): Check other models whether language generation is also applicable
			lowercase__   :	Tuple  			=      (
			    {
			        """feature-extraction""": TFXLMModel,
			        """fill-mask""": TFXLMWithLMHeadModel,
			        """question-answering""": TFXLMForQuestionAnsweringSimple,
			        """text-classification""": TFXLMForSequenceClassification,
			        """text-generation""": TFXLMWithLMHeadModel,
			        """token-classification""": TFXLMForTokenClassification,
			        """zero-shot""": TFXLMForSequenceClassification,
			    }
			    if is_tf_available()
			    else {}
			)
			lowercase__   :	int  			=      False
			lowercase__   :	int  			=      False





			def 		snake_case__      (					self						:							Optional[int]      ,  lowercase						:							List[str]      ,  lowercase						:							Optional[Any]      ,  lowercase						:							Optional[int]      ,  lowercase						:							List[str]      ,  lowercase						:							Dict )    ->       str:
									"""simple docstring"""







									if (
									    pipeline_test_casse_name == "QAPipelineTests"
									    and tokenizer_name is not None
									    and not tokenizer_name.endswith("""Fast""" )
									):
															# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
															# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
															# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
															return True

									return False





			def 		snake_case__      (					self						:							Tuple )    ->       List[str]:
									"""simple docstring"""







									__lowercase											=					TFXLMModelTester(self )
									__lowercase											=					ConfigTester(self      ,  config_class=lowercase      ,  emb_dim=37 )





			def 		snake_case__      (					self						:							List[str] )    ->       List[str]:
									"""simple docstring"""







									self.config_tester.run_common_tests()





			def 		snake_case__      (					self						:							List[Any] )    ->       List[Any]:
									"""simple docstring"""







									__lowercase											=					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_model(*lowercase )





			def 		snake_case__      (					self						:							str )    ->       str:
									"""simple docstring"""







									__lowercase											=					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_lm_head(*lowercase )





			def 		snake_case__      (					self						:							Optional[int] )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_qa(*lowercase )





			def 		snake_case__      (					self						:							Optional[Any] )    ->       str:
									"""simple docstring"""







									__lowercase											=					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_sequence_classif(*lowercase )





			def 		snake_case__      (					self						:							Union[str, Any] )    ->       int:
									"""simple docstring"""







									__lowercase											=					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_for_token_classification(*lowercase )





			def 		snake_case__      (					self						:							Optional[Any] )    ->       Tuple:
									"""simple docstring"""







									__lowercase											=					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_for_multiple_choice(*lowercase )







			@slow
			def 		snake_case__      (					self						:							Optional[int] )    ->       Optional[int]:
									"""simple docstring"""







									for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															__lowercase											=					TFXLMModel.from_pretrained(lowercase )
															self.assertIsNotNone(lowercase )





@require_tf
class       _lowerCAmelCase (			unittest.TestCase      ):






			"""simple docstring"""





			@slow
			def 		snake_case__      (					self						:							int )    ->       List[Any]:
									"""simple docstring"""







									__lowercase											=					TFXLMWithLMHeadModel.from_pretrained("""xlm-mlm-en-2048""" )
									__lowercase											=					tf.convert_to_tensor([[14, 447]]      ,  dtype=tf.intaa )  # the president
									__lowercase											=					[
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									]  # the president the president the president the president the president the president the president the president the president the president
									# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
									__lowercase											=					model.generate(lowercase      ,  do_sample=lowercase )
									self.assertListEqual(output_ids[0].numpy().tolist()      ,  lowercase )





