





'''simple docstring'''

import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase							:	List[Any]			=get_tests_dir("fixtures/spiece.model")






@require_sentencepiece
@require_tokenizers
class    UpperCAmelCase	(       UpperCamelCase__			,       unittest.TestCase				):
 __lowercase   		=				DebertaVaTokenizer
 __lowercase   		=				DebertaVaTokenizerFast
 __lowercase   		=				True
 __lowercase   		=				True
 def      UpperCAmelCase_							(				self      :Optional[int]							)->							int:
     super().setUp()

     # We have a SentencePiece fixture for testing
     A__				=			DebertaVaTokenizer(lowercase_						,       unk_token="<unk>"							)
     tokenizer.save_pretrained(self.tmpdirname							)
 def      UpperCAmelCase_							(				self      :Dict						,       lowercase_      :str							)->							Dict:
     A__				=			"this is a test"
     A__				=			"this is a test"
     return input_text, output_text
 def      UpperCAmelCase_							(				self      :Union[str, Any]							)->							Optional[int]:
     A__				=			"<pad>"
     A__				=			0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_							)						,       lowercase_							)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_							)						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :Any							)->							Any:
     A__				=			list(self.get_tokenizer().get_vocab().keys()							)
     self.assertEqual(vocab_keys[0]						,       "<pad>"							)
     self.assertEqual(vocab_keys[1]						,       "<unk>"							)
     self.assertEqual(vocab_keys[-1]						,       "[PAD]"							)
     self.assertEqual(len(lowercase_							)						,       3_00_01							)
 def      UpperCAmelCase_							(				self      :Optional[int]							)->							Union[str, Any]:
     self.assertEqual(self.get_tokenizer().vocab_size						,       3_00_00							)
 def      UpperCAmelCase_							(				self      :Tuple							)->							int:
     # fmt: off
     A__				=			" \tHeLLo!how  \n Are yoU?  "
     A__				=			["▁hello", "!", "how", "▁are", "▁you", "?"]
     # fmt: on

     A__				=			DebertaVaTokenizer(lowercase_						,       do_lower_case=lowercase_							)
     A__				=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			DebertaVaTokenizerFast(lowercase_						,       do_lower_case=lowercase_							)
     A__				=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)
 @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."							)
 def      UpperCAmelCase_							(				self      :Tuple							)->							Dict:
     pass
 @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."							)
 def      UpperCAmelCase_							(				self      :List[Any]							)->							Tuple:
     pass
 def      UpperCAmelCase_							(				self      :List[Any]							)->							Optional[int]:
     # fmt: off
     A__				=			"I was born in 92000, and this is falsé."
     A__				=			["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
     # fmt: on

     A__				=			DebertaVaTokenizer(lowercase_						,       split_by_punct=lowercase_							)
     A__				=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			DebertaVaTokenizerFast(lowercase_						,       split_by_punct=lowercase_							)
     A__				=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :int							)->							Optional[Any]:
     # fmt: off
     A__				=			"I was born in 92000, and this is falsé."
     A__				=			["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
     # fmt: on

     A__				=			DebertaVaTokenizer(lowercase_						,       do_lower_case=lowercase_						,       split_by_punct=lowercase_							)
     A__				=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			DebertaVaTokenizerFast(lowercase_						,       do_lower_case=lowercase_						,       split_by_punct=lowercase_							)
     A__				=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)
     self.assertListEqual(lowercase_						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :List[Any]							)->							Union[str, Any]:
     # fmt: off
     A__				=			"I was born in 92000, and this is falsé."
     A__				=			["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
     # fmt: on

     A__				=			DebertaVaTokenizer(lowercase_						,       do_lower_case=lowercase_						,       split_by_punct=lowercase_							)
     A__				=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			DebertaVaTokenizerFast(lowercase_						,       do_lower_case=lowercase_						,       split_by_punct=lowercase_							)
     A__				=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :str							)->							Any:
     # fmt: off
     A__				=			"I was born in 92000, and this is falsé."
     A__				=			["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
     # fmt: on

     A__				=			DebertaVaTokenizer(lowercase_						,       do_lower_case=lowercase_						,       split_by_punct=lowercase_							)
     A__				=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			DebertaVaTokenizerFast(lowercase_						,       do_lower_case=lowercase_						,       split_by_punct=lowercase_							)
     A__				=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :int							)->							Optional[int]:
     # fmt: off
     A__				=			" \tHeLLo!how  \n Are yoU?  "
     A__				=			["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
     # fmt: on

     A__				=			DebertaVaTokenizer(lowercase_						,       do_lower_case=lowercase_						,       split_by_punct=lowercase_							)
     A__				=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			DebertaVaTokenizerFast(lowercase_						,       do_lower_case=lowercase_						,       split_by_punct=lowercase_							)
     A__				=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)

     self.assertListEqual(lowercase_						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :List[Any]							)->							str:
     A__				=			self.get_tokenizer()
     A__				=			self.get_rust_tokenizer()

     A__				=			"I was born in 92000, and this is falsé."

     A__				=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)
     A__				=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     A__				=			rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			self.get_rust_tokenizer()
     A__				=			tokenizer.encode(lowercase_							)
     A__				=			rust_tokenizer.encode(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :List[str]							)->							Optional[Any]:
     A__				=			"This is a test"
     A__				=			[13, 1, 43_98, 25, 21, 12_89]
     A__				=			["▁", "T", "his", "▁is", "▁a", "▁test"]
     A__				=			["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

     A__				=			DebertaVaTokenizer(lowercase_						,       keep_accents=lowercase_							)
     A__				=			DebertaVaTokenizerFast(lowercase_						,       keep_accents=lowercase_							)

     A__				=			tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
     A__				=			tokenizer.tokenize(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
     A__				=			tokenizer.convert_ids_to_tokens(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
     A__				=			rust_tokenizer.tokenize(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
     A__				=			rust_tokenizer.convert_ids_to_tokens(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     # fmt: off
     A__				=			"I was born in 92000, and this is falsé."
     A__				=			[13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
     A__				=			["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
     A__				=			["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
     # fmt: on

     A__				=			tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
     A__				=			tokenizer.tokenize(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
     A__				=			tokenizer.convert_ids_to_tokens(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
     A__				=			rust_tokenizer.tokenize(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
     A__				=			rust_tokenizer.convert_ids_to_tokens(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :List[Any]							)->							List[str]:
     A__				=			DebertaVaTokenizer(lowercase_							)

     A__				=			tokenizer.encode("sequence builders"							)
     A__				=			tokenizer.encode("multi-sequence build"							)

     A__				=			tokenizer.build_inputs_with_special_tokens(lowercase_							)
     A__				=			tokenizer.build_inputs_with_special_tokens(lowercase_						,       lowercase_							)

     self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]						,       lowercase_							)
     self.assertEqual(
         [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]						,       lowercase_						,       )



 @slow
 def      UpperCAmelCase_							(				self      :Union[str, Any]							)->							Any:
     # fmt: off
     A__				=			{"input_ids": [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=lowercase_						,       model_name="microsoft/deberta-v2-xlarge"						,       revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"						,       )










'''simple docstring'''

# NOTE: This file is deprecated and will be removed in a future version.
# It only exists so that temporarely `from diffusers.pipelines import DiffusionPipeline` works
from ...utils import deprecate
from ..controlnet.multicontrolnet import MultiControlNetModel  # noqa: F401
from ..controlnet.pipeline_controlnet import StableDiffusionControlNetPipeline  # noqa: F401


deprecate(
    "stable diffusion controlnet",
    "0.22.0",
    "Importing `StableDiffusionControlNetPipeline` or `MultiControlNetModel` from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet is deprecated. Please import `from diffusers import StableDiffusionControlNetPipeline` instead.",
    standard_warn=False,
    stacklevel=3,
)



