


'''simple docstring'''


import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__					=   get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
class 		lowerCAmelCase__	(	UpperCAmelCase__			,				unittest.TestCase		):
			lowerCAmelCase       :		List[Any]				      =   XLMProphetNetTokenizer
			lowerCAmelCase       :		Optional[Any]				      =   False
			lowerCAmelCase       :		Tuple				      =   True




			def 							lowerCAmelCase__			(						self     :   str       )      ->List[str]:



					'''simple docstring'''







					super().setUp()

					# We have a SentencePiece fixture for testing
					_UpperCAmelCase						:				List[str]	    =					XLMProphetNetTokenizer(lowerCamelCase__		,     keep_accents=lowerCamelCase__       )
					tokenizer.save_pretrained(self.tmpdirname       )




			def 							lowerCAmelCase__			(						self     :   List[str]       )      ->Dict:



					'''simple docstring'''







					_UpperCAmelCase						:				Tuple	    =					"[PAD]"
					_UpperCAmelCase						:				str	    =					0

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase__       )		,     lowerCamelCase__       )
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase__       )		,     lowerCamelCase__       )




			def 							lowerCAmelCase__			(						self     :   str       )      ->Optional[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				str	    =					list(self.get_tokenizer().get_vocab().keys()       )

					self.assertEqual(vocab_keys[0]		,     "[PAD]"       )
					self.assertEqual(vocab_keys[1]		,     "[CLS]"       )
					self.assertEqual(vocab_keys[-1]		,     "j"       )
					self.assertEqual(len(lowerCamelCase__       )		,     10_12       )




			def 							lowerCAmelCase__			(						self     :   str       )      ->int:



					'''simple docstring'''







					self.assertEqual(self.get_tokenizer().vocab_size		,     10_12       )




			def 							lowerCAmelCase__			(						self     :   Union[str, Any]       )      ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase						:				Union[str, Any]	    =					XLMProphetNetTokenizer(lowerCamelCase__		,     keep_accents=lowerCamelCase__       )

					_UpperCAmelCase						:				Optional[Any]	    =					tokenizer.tokenize("This is a test"       )
					self.assertListEqual(lowerCamelCase__		,     ["▁This", "▁is", "▁a", "▁t", "est"]       )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(lowerCamelCase__       )		,     [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]		,     )

					_UpperCAmelCase						:				Dict	    =					tokenizer.tokenize("I was born in 92000, and this is falsé."       )
					self.assertListEqual(
					    lowerCamelCase__		,     [
					        SPIECE_UNDERLINE + "I",
					        SPIECE_UNDERLINE + "was",
					        SPIECE_UNDERLINE + "b",
					        "or",
					        "n",
					        SPIECE_UNDERLINE + "in",
					        SPIECE_UNDERLINE + "",
					        "9",
					        "2",
					        "0",
					        "0",
					        "0",
					        ",",
					        SPIECE_UNDERLINE + "and",
					        SPIECE_UNDERLINE + "this",
					        SPIECE_UNDERLINE + "is",
					        SPIECE_UNDERLINE + "f",
					        "al",
					        "s",
					        "é",
					        ".",
					    ]		,     )
					_UpperCAmelCase						:				Union[str, Any]	    =					tokenizer.convert_tokens_to_ids(lowerCamelCase__       )
					self.assertListEqual(
					    lowerCamelCase__		,     [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
					    ]		,     )

					_UpperCAmelCase						:				Dict	    =					tokenizer.convert_ids_to_tokens(lowerCamelCase__       )
					self.assertListEqual(
					    lowerCamelCase__		,     [
					        SPIECE_UNDERLINE + "I",
					        SPIECE_UNDERLINE + "was",
					        SPIECE_UNDERLINE + "b",
					        "or",
					        "n",
					        SPIECE_UNDERLINE + "in",
					        SPIECE_UNDERLINE + "",
					        "[UNK]",
					        "2",
					        "0",
					        "0",
					        "0",
					        ",",
					        SPIECE_UNDERLINE + "and",
					        SPIECE_UNDERLINE + "this",
					        SPIECE_UNDERLINE + "is",
					        SPIECE_UNDERLINE + "f",
					        "al",
					        "s",
					        "[UNK]",
					        ".",
					    ]		,     )




			@cached_property
			def 							lowerCAmelCase__			(						self     :   List[str]       )      ->int:



					'''simple docstring'''







					return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"       )




			@slow
			def 							lowerCAmelCase__			(						self     :   Dict       )      ->Optional[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					"Hello World!"
					_UpperCAmelCase						:				int	    =					[3_53_89, 66_72, 49, 2]
					self.assertListEqual(lowerCamelCase__		,     self.big_tokenizer.encode(lowerCamelCase__       )       )






			@slow
			def 							lowerCAmelCase__			(						self     :   List[str]       )      ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				int	    =					{"input_ids": [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=lowerCamelCase__		,     model_name="microsoft/xprophetnet-large-wiki100-cased"		,     revision="1acad1643ddd54a44df6a1b797ada8373685d90e"		,     )







'''simple docstring'''


import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


lowerCamelCase__					=   logging.get_logger(__name__)

lowerCamelCase__					=   {
    'ut/deta': 'https://huggingface.co/ut/deta/resolve/main/config.json',
}






class 		lowerCAmelCase__	(	UpperCAmelCase__		):
			lowerCAmelCase       :		Any				      =   "deta"
			lowerCAmelCase       :		Union[str, Any]				      =   {
			    "hidden_size": "d_model",
			    "num_attention_heads": "encoder_attention_heads",
			}




			def __init__(						self     :   Optional[Any]		,     lowerCamelCase__     :   str=None		,     lowerCamelCase__     :   str=9_00		,     lowerCamelCase__     :   Any=20_48		,     lowerCamelCase__     :   Optional[int]=6		,     lowerCamelCase__     :   str=20_48		,     lowerCamelCase__     :   Dict=8		,     lowerCamelCase__     :   Any=6		,     lowerCamelCase__     :   Union[str, Any]=10_24		,     lowerCamelCase__     :   Optional[int]=8		,     lowerCamelCase__     :   List[Any]=0.0		,     lowerCamelCase__     :   Tuple=True		,     lowerCamelCase__     :   int="relu"		,     lowerCamelCase__     :   str=2_56		,     lowerCamelCase__     :   Dict=0.1		,     lowerCamelCase__     :   Dict=0.0		,     lowerCamelCase__     :   Dict=0.0		,     lowerCamelCase__     :   Optional[int]=0.0_2		,     lowerCamelCase__     :   List[Any]=1.0		,     lowerCamelCase__     :   Optional[int]=True		,     lowerCamelCase__     :   Any=False		,     lowerCamelCase__     :   Any="sine"		,     lowerCamelCase__     :   str=5		,     lowerCamelCase__     :   Optional[int]=4		,     lowerCamelCase__     :   int=4		,     lowerCamelCase__     :   Optional[Any]=True		,     lowerCamelCase__     :   Dict=3_00		,     lowerCamelCase__     :   int=True		,     lowerCamelCase__     :   Dict=True		,     lowerCamelCase__     :   Union[str, Any]=1		,     lowerCamelCase__     :   Any=5		,     lowerCamelCase__     :   List[Any]=2		,     lowerCamelCase__     :   Union[str, Any]=1		,     lowerCamelCase__     :   str=1		,     lowerCamelCase__     :   Union[str, Any]=5		,     lowerCamelCase__     :   str=2		,     lowerCamelCase__     :   Optional[Any]=0.1		,     lowerCamelCase__     :   Union[str, Any]=0.2_5		,     **lowerCamelCase__     :   Optional[Any]		,     )      ->List[str]:



					'''simple docstring'''







					if backbone_config is None:
							logger.info("`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone."       )
							_UpperCAmelCase						:				int	    =					CONFIG_MAPPING["resnet"](out_features=["stage2", "stage3", "stage4"]       )
					else:
							if isinstance(lowerCamelCase__		,     lowerCamelCase__       ):
									_UpperCAmelCase						:				Any	    =					backbone_config.pop("model_type"       )
									_UpperCAmelCase						:				Optional[int]	    =					CONFIG_MAPPING[backbone_model_type]
									_UpperCAmelCase						:				List[str]	    =					config_class.from_dict(lowerCamelCase__       )

					_UpperCAmelCase						:				Optional[Any]	    =					backbone_config
					_UpperCAmelCase						:				Optional[int]	    =					num_queries
					_UpperCAmelCase						:				Union[str, Any]	    =					max_position_embeddings
					_UpperCAmelCase						:				Union[str, Any]	    =					d_model
					_UpperCAmelCase						:				str	    =					encoder_ffn_dim
					_UpperCAmelCase						:				Optional[int]	    =					encoder_layers
					_UpperCAmelCase						:				int	    =					encoder_attention_heads
					_UpperCAmelCase						:				Union[str, Any]	    =					decoder_ffn_dim
					_UpperCAmelCase						:				Tuple	    =					decoder_layers
					_UpperCAmelCase						:				Union[str, Any]	    =					decoder_attention_heads
					_UpperCAmelCase						:				Any	    =					dropout
					_UpperCAmelCase						:				List[str]	    =					attention_dropout
					_UpperCAmelCase						:				Union[str, Any]	    =					activation_dropout
					_UpperCAmelCase						:				Optional[int]	    =					activation_function
					_UpperCAmelCase						:				str	    =					init_std
					_UpperCAmelCase						:				Tuple	    =					init_xavier_std
					_UpperCAmelCase						:				Optional[Any]	    =					encoder_layerdrop
					_UpperCAmelCase						:				int	    =					auxiliary_loss
					_UpperCAmelCase						:				Union[str, Any]	    =					position_embedding_type
					# deformable attributes
					_UpperCAmelCase						:				List[Any]	    =					num_feature_levels
					_UpperCAmelCase						:				List[Any]	    =					encoder_n_points
					_UpperCAmelCase						:				Tuple	    =					decoder_n_points
					_UpperCAmelCase						:				Optional[int]	    =					two_stage
					_UpperCAmelCase						:				Dict	    =					two_stage_num_proposals
					_UpperCAmelCase						:				int	    =					with_box_refine
					_UpperCAmelCase						:				str	    =					assign_first_stage
					if two_stage is True and with_box_refine is False:
							raise ValueError("If two_stage is True, with_box_refine must be True."       )
					# Hungarian matcher
					_UpperCAmelCase						:				Optional[int]	    =					class_cost
					_UpperCAmelCase						:				Dict	    =					bbox_cost
					_UpperCAmelCase						:				int	    =					giou_cost
					# Loss coefficients
					_UpperCAmelCase						:				int	    =					mask_loss_coefficient
					_UpperCAmelCase						:				List[Any]	    =					dice_loss_coefficient
					_UpperCAmelCase						:				Dict	    =					bbox_loss_coefficient
					_UpperCAmelCase						:				int	    =					giou_loss_coefficient
					_UpperCAmelCase						:				Optional[Any]	    =					eos_coefficient
					_UpperCAmelCase						:				int	    =					focal_alpha
					super().__init__(is_encoder_decoder=lowerCamelCase__		,     **lowerCamelCase__       )




			@property
			def 							lowerCAmelCase__			(						self     :   int       )      ->int:



					'''simple docstring'''







					return self.encoder_attention_heads




			@property
			def 							lowerCAmelCase__			(						self     :   List[Any]       )      ->int:



					'''simple docstring'''







					return self.d_model






			def 							lowerCAmelCase__			(						self     :   Union[str, Any]       )      ->int:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[int]	    =					copy.deepcopy(self.__dict__       )
					_UpperCAmelCase						:				Tuple	    =					self.backbone_config.to_dict()
					_UpperCAmelCase						:				List[Any]	    =					self.__class__.model_type
					return output



