


import unittest

import numpy as np
import torch

from diffusers import KarrasVePipeline, KarrasVeScheduler, UNetaDModel
from diffusers.utils.testing_utils import enable_full_determinism, require_torch, slow, torch_device


enable_full_determinism()






class     UpperCAmelCase__       (					unittest.TestCase		):



    """simple docstring"""






    @property
    def _a					(			self    )  ->       List[str]:
         torch.manual_seed(0    )
         __UpperCamelCase    							=UNetaDModel(
             block_out_channels=(32, 64)					,   layers_per_block=2					,   sample_size=32					,   in_channels=3					,   out_channels=3					,   down_block_types=('DownBlock2D', 'AttnDownBlock2D')					,   up_block_types=('AttnUpBlock2D', 'UpBlock2D')					,   )
         return model






    def _a					(			self    )  ->       Optional[int]:
         __UpperCamelCase    							=self.dummy_uncond_unet
         __UpperCamelCase    							=KarrasVeScheduler()

         __UpperCamelCase    							=KarrasVePipeline(unet=A_					,   scheduler=A_    )
         pipe.to(A_    )
         pipe.set_progress_bar_config(disable=A_    )

         __UpperCamelCase    							=torch.manual_seed(0    )
         __UpperCamelCase    							=pipe(num_inference_steps=2					,   generator=A_					,   output_type='numpy'    ).images

         __UpperCamelCase    							=torch.manual_seed(0    )
         __UpperCamelCase    							=pipe(num_inference_steps=2					,   generator=A_					,   output_type='numpy'					,   return_dict=A_    )[0]

         __UpperCamelCase    							=image[0, -3:, -3:, -1]
         __UpperCamelCase    							=image_from_tuple[0, -3:, -3:, -1]

         assert image.shape == (1, 32, 32, 3)
         __UpperCamelCase    							=np.array([0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0]    )

         assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1E-2
         assert np.abs(image_from_tuple_slice.flatten() - expected_slice    ).max() < 1E-2






@slow
@require_torch
class     UpperCAmelCase__       (					unittest.TestCase		):



    """simple docstring"""






    def _a					(			self    )  ->       Union[str, Any]:
         __UpperCamelCase    							='google/ncsnpp-celebahq-256'
         __UpperCamelCase    							=UNetaDModel.from_pretrained(A_    )
         __UpperCamelCase    							=KarrasVeScheduler()

         __UpperCamelCase    							=KarrasVePipeline(unet=A_					,   scheduler=A_    )
         pipe.to(A_    )
         pipe.set_progress_bar_config(disable=A_    )

         __UpperCamelCase    							=torch.manual_seed(0    )
         __UpperCamelCase    							=pipe(num_inference_steps=20					,   generator=A_					,   output_type='numpy'    ).images

         __UpperCamelCase    							=image[0, -3:, -3:, -1]
         assert image.shape == (1, 256, 256, 3)
         __UpperCamelCase    							=np.array([0.578, 0.5811, 0.5924, 0.5809, 0.587, 0.5886, 0.5861, 0.5802, 0.586]    )

         assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1E-2








"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class     snake_case	(				__snake_case,							unittest.TestCase  ):
				SCREAMING_SNAKE_CASE_   :      List[Any]         =  RoCBertTokenizer
				SCREAMING_SNAKE_CASE_   :      Any         =  None
				SCREAMING_SNAKE_CASE_   :      Any         =  False
				SCREAMING_SNAKE_CASE_   :      Dict         =  True
				SCREAMING_SNAKE_CASE_   :      Optional[int]         =  filter_non_english

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						super().setUp()

						__lowerCAmelCase:       Optional[Any]          =			["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
						__lowerCAmelCase:       List[Any]          =			{}
						__lowerCAmelCase:       Dict          =			{}
						for i, value in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       List[Any]          =			i
								__lowerCAmelCase:       Union[str, Any]          =			i
						__lowerCAmelCase:       List[str]          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["vocab_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_shape_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_pronunciation_file"])
						with open(self.vocab_file     ,  "w"     ,  encoding="utf-8") as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
						with open(self.word_shape_file     ,  "w"     ,  encoding="utf-8") as word_shape_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)
						with open(self.word_pronunciation_file     ,  "w"     ,  encoding="utf-8") as word_pronunciation_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)

				def 							lowercase_     (	self							:				Any)->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.tokenize("你好[SEP]你是谁")
						self.assertListEqual(UpperCamelCase__     ,  ["你", "好", "[SEP]", "你", "是", "谁"])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])

				def 							lowercase_     (	self							:				Optional[Any])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")     ,  ["ah", "\u535A", "\u63A8", "zz"])

				def 							lowercase_     (	self							:				str)->   Dict:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["hello", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Optional[int])->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hällo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["h\u00E9llo"])

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				str)->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Any)->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[int])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[int]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HäLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[Any])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HaLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Tuple)->   str:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  never_split=["[UNK]"])

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])

				def 							lowercase_     (	self							:				List[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

						__lowerCAmelCase:       int          =			{}
						for i, token in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       Optional[Any]          =			i
						__lowerCAmelCase:       str          =			RoCBertWordpieceTokenizer(vocab=UpperCamelCase__     ,  unk_token="[UNK]")

						self.assertListEqual(tokenizer.tokenize("")     ,  [])

						self.assertListEqual(tokenizer.tokenize("unwanted running")     ,  ["un", "##want", "##ed", "runn", "##ing"])

						self.assertListEqual(tokenizer.tokenize("unwantedX running")     ,  ["[UNK]", "runn", "##ing"])

				def 							lowercase_     (	self							:				Optional[Any])->   Dict:



						'''simple docstring'''


						self.assertTrue(_is_whitespace(" "))
						self.assertTrue(_is_whitespace("\t"))
						self.assertTrue(_is_whitespace("\r"))
						self.assertTrue(_is_whitespace("\n"))
						self.assertTrue(_is_whitespace("\u00A0"))

						self.assertFalse(_is_whitespace("A"))
						self.assertFalse(_is_whitespace("-"))

				def 							lowercase_     (	self							:				Dict)->   Optional[int]:



						'''simple docstring'''


						self.assertTrue(_is_control("\u0005"))

						self.assertFalse(_is_control("A"))
						self.assertFalse(_is_control(" "))
						self.assertFalse(_is_control("\t"))
						self.assertFalse(_is_control("\r"))

				def 							lowercase_     (	self							:				Union[str, Any])->   str:



						'''simple docstring'''


						self.assertTrue(_is_punctuation("-"))
						self.assertTrue(_is_punctuation("$"))
						self.assertTrue(_is_punctuation("`"))
						self.assertTrue(_is_punctuation("."))

						self.assertFalse(_is_punctuation("A"))
						self.assertFalse(_is_punctuation(" "))

				def 							lowercase_     (	self							:				Dict)->   int:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

						if self.test_rust_tokenizer:
								__lowerCAmelCase:       Any          =			self.get_rust_tokenizer()
								self.assertListEqual(
								    [rust_tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

				def 							lowercase_     (	self							:				Dict)->   Any:



						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       str          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
										__lowerCAmelCase:       Tuple          =			tokenizer_r.encode_plus(
										    UpperCamelCase__     ,  return_attention_mask=UpperCamelCase__     ,  return_token_type_ids=UpperCamelCase__     ,  return_offsets_mapping=UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__     ,  )

										__lowerCAmelCase:       str          =			tokenizer_r.do_lower_case if hasattr(UpperCamelCase__     ,  "do_lower_case") else False
										__lowerCAmelCase:       List[Any]          =			(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "Allen"),
										        ((2_1, 2_3), "##NL"),
										        ((2_3, 2_4), "##P"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "allen"),
										        ((2_1, 2_3), "##nl"),
										        ((2_3, 2_4), "##p"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]     ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
										self.assertEqual([e[0] for e in expected_results]     ,  tokens["offset_mapping"])

				def 							lowercase_     (	self							:				Union[str, Any])->   Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			["的", "人", "有"]
						__lowerCAmelCase:       int          =			"".join(UpperCamelCase__)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       Tuple          =			True
										__lowerCAmelCase:       str          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Union[str, Any]          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       List[Any]          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Any          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       List[str]          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

										__lowerCAmelCase:       int          =			False
										__lowerCAmelCase:       Any          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       str          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowerCAmelCase:       Dict          =			[
										    f"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCamelCase__)
										]
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

				@slow
				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       str          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Dict          =			tokenizer.encode("你好"     ,  add_special_tokens=UpperCamelCase__)
						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.encode("你是谁"     ,  add_special_tokens=UpperCamelCase__)

						__lowerCAmelCase:       Tuple          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__)
						__lowerCAmelCase:       List[Any]          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     ,  UpperCamelCase__)

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]




				def 							lowercase_     (	self							:				Tuple)->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			self.get_tokenizers(do_lower_case=UpperCamelCase__)
						for tokenizer in tokenizers:
								with self.subTest(f"{tokenizer.__class__.__name__}"):
										__lowerCAmelCase:       str          =			"你好，你是谁"
										__lowerCAmelCase:       Dict          =			tokenizer.tokenize(UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			tokenizer.convert_tokens_to_ids(UpperCamelCase__)
										__lowerCAmelCase:       Optional[Any]          =			tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			tokenizer.prepare_for_model(
										    UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			tokenizer.encode_plus(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										self.assertEqual(UpperCamelCase__     ,  UpperCamelCase__)


