
import tempfile
import unittest

from diffusers import (
    DDIMScheduler,
    DDPMScheduler,
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    PNDMScheduler,
    logging,
)
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.utils.testing_utils import CaptureLogger






class 						UpperCAmelCase	(		A_				):
   A__      :	List[Any]		  =  "config.json"



   @register_to_config
   def __init__(self	:      int					,		snake_case__	:      Any=2					,		snake_case__	:      Optional[Any]=5					,		snake_case__	:      str=(2, 5)					,		snake_case__	:      List[Any]="for diffusion"					,		snake_case__	:      int=[1, 3]					,		)	->			Tuple:


          '''simple docstring'''


          pass






class 						UpperCAmelCase	(		A_				):
   A__      :	Dict		  =  "config.json"



   @register_to_config
   def __init__(self	:      Union[str, Any]					,		snake_case__	:      List[str]=2					,		snake_case__	:      Any=5					,		snake_case__	:      Optional[int]=(2, 5)					,		snake_case__	:      Dict="for diffusion"					,		snake_case__	:      Optional[int]=[1, 3]					,		)	->			Any:


          '''simple docstring'''


          pass






class 						UpperCAmelCase	(		A_				):
   A__      :	Union[str, Any]		  =  "config.json"



   @register_to_config
   def __init__(self	:      List[Any]					,		snake_case__	:      Optional[int]=2					,		snake_case__	:      int=5					,		snake_case__	:      Union[str, Any]=(2, 5)					,		snake_case__	:      Union[str, Any]="for diffusion"					,		snake_case__	:      List[str]=[1, 3]					,		snake_case__	:      Optional[Any]=[1, 3]					,		)	->			Tuple:


          '''simple docstring'''


          pass






class 						UpperCAmelCase	(		A_				):
   A__      :	Optional[int]		  =  "config.json"



   @register_to_config
   def __init__(self	:      List[str]					,		snake_case__	:      str=2					,		snake_case__	:      Tuple=5					,		snake_case__	:      Dict=(2, 5)					,		snake_case__	:      Tuple="for diffusion"					,		snake_case__	:      Tuple=[1, 5]					,		snake_case__	:      Tuple=[5, 4]					,		)	->			Any:


          '''simple docstring'''


          pass






class 						UpperCAmelCase	(		unittest.TestCase				):



   def        _SCREAMING_SNAKE_CASE     (self	:      Tuple				)	->			Tuple:


          '''simple docstring'''


          with self.assertRaises(snake_case__				):
                 ConfigMixin.load_config("dummy_path"				)



   def        _SCREAMING_SNAKE_CASE     (self	:      Any				)	->			Tuple:


          '''simple docstring'''


          snake_case :     Dict           =   SampleObject()
          snake_case :     Tuple           =   obj.config
          assert config["a"] == 2
          assert config["b"] == 5
          assert config["c"] == (2, 5)
          assert config["d"] == "for diffusion"
          assert config["e"] == [1, 3]

          # init ignore private arguments
          snake_case :     Optional[int]           =   SampleObject(_name_or_path="lalala"				)
          snake_case :     List[Any]           =   obj.config
          assert config["a"] == 2
          assert config["b"] == 5
          assert config["c"] == (2, 5)
          assert config["d"] == "for diffusion"
          assert config["e"] == [1, 3]

          # can override default
          snake_case :     int           =   SampleObject(c=6				)
          snake_case :     Union[str, Any]           =   obj.config
          assert config["a"] == 2
          assert config["b"] == 5
          assert config["c"] == 6
          assert config["d"] == "for diffusion"
          assert config["e"] == [1, 3]

          # can use positional arguments.
          snake_case :     Tuple           =   SampleObject(1					,		c=6				)
          snake_case :     Tuple           =   obj.config
          assert config["a"] == 1
          assert config["b"] == 5
          assert config["c"] == 6
          assert config["d"] == "for diffusion"
          assert config["e"] == [1, 3]



   def        _SCREAMING_SNAKE_CASE     (self	:      int				)	->			List[Any]:


          '''simple docstring'''


          snake_case :     Dict           =   SampleObject()
          snake_case :     str           =   obj.config

          assert config["a"] == 2
          assert config["b"] == 5
          assert config["c"] == (2, 5)
          assert config["d"] == "for diffusion"
          assert config["e"] == [1, 3]

          with tempfile.TemporaryDirectory() as tmpdirname:
                 obj.save_config(snake_case__				)
                 snake_case :     List[Any]           =   SampleObject.from_config(SampleObject.load_config(snake_case__				)				)
                 snake_case :     Any           =   new_obj.config

          # unfreeze configs
          snake_case :     str           =   dict(snake_case__				)
          snake_case :     int           =   dict(snake_case__				)

          assert config.pop("c"				) == (2, 5)  # instantiated as tuple
          assert new_config.pop("c"				) == [2, 5]  # saved & loaded as list because of json
          config.pop("_use_default_values"				)
          assert config == new_config



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[Any]				)	->			int:


          '''simple docstring'''


          snake_case :     Optional[int]           =   logging.get_logger("diffusers.configuration_utils"				)
          # 30 for warning
          logger.setLevel(30				)

          with CaptureLogger(snake_case__				) as cap_logger:
                 snake_case :     Dict           =   DDIMScheduler.from_pretrained(
                     "hf-internal-testing/tiny-stable-diffusion-torch"					,		subfolder="scheduler"				)

          assert ddim.__class__ == DDIMScheduler
          # no warning should be thrown
          assert cap_logger.out == ""



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[Any]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     Dict           =   logging.get_logger("diffusers.configuration_utils"				)
          # 30 for warning
          logger.setLevel(30				)

          with CaptureLogger(snake_case__				) as cap_logger:
                 snake_case :     Tuple           =   EulerDiscreteScheduler.from_pretrained(
                     "hf-internal-testing/tiny-stable-diffusion-torch"					,		subfolder="scheduler"				)

          assert euler.__class__ == EulerDiscreteScheduler
          # no warning should be thrown
          assert cap_logger.out == ""



   def        _SCREAMING_SNAKE_CASE     (self	:      Tuple				)	->			List[Any]:


          '''simple docstring'''


          snake_case :     Union[str, Any]           =   logging.get_logger("diffusers.configuration_utils"				)
          # 30 for warning
          logger.setLevel(30				)

          with CaptureLogger(snake_case__				) as cap_logger:
                 snake_case :     List[str]           =   EulerAncestralDiscreteScheduler.from_pretrained(
                     "hf-internal-testing/tiny-stable-diffusion-torch"					,		subfolder="scheduler"				)

          assert euler.__class__ == EulerAncestralDiscreteScheduler
          # no warning should be thrown
          assert cap_logger.out == ""



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[Any]				)	->			Any:


          '''simple docstring'''


          snake_case :     int           =   logging.get_logger("diffusers.configuration_utils"				)
          # 30 for warning
          logger.setLevel(30				)

          with CaptureLogger(snake_case__				) as cap_logger:
                 snake_case :     List[str]           =   PNDMScheduler.from_pretrained(
                     "hf-internal-testing/tiny-stable-diffusion-torch"					,		subfolder="scheduler"				)

          assert pndm.__class__ == PNDMScheduler
          # no warning should be thrown
          assert cap_logger.out == ""



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			Optional[Any]:


          '''simple docstring'''


          snake_case :     Union[str, Any]           =   logging.get_logger("diffusers.configuration_utils"				)
          # 30 for warning
          logger.setLevel(30				)

          with CaptureLogger(snake_case__				) as cap_logger:
                 snake_case :     Any           =   DDPMScheduler.from_pretrained(
                     "hf-internal-testing/tiny-stable-diffusion-torch"					,		subfolder="scheduler"					,		prediction_type="sample"					,		beta_end=8					,		)

          with CaptureLogger(snake_case__				) as cap_logger_a:
                 snake_case :     List[Any]           =   DDPMScheduler.from_pretrained("google/ddpm-celebahq-256"					,		beta_start=88				)

          assert ddpm.__class__ == DDPMScheduler
          assert ddpm.config.prediction_type == "sample"
          assert ddpm.config.beta_end == 8
          assert ddpm_a.config.beta_start == 88

          # no warning should be thrown
          assert cap_logger.out == ""
          assert cap_logger_a.out == ""



   def        _SCREAMING_SNAKE_CASE     (self	:      Tuple				)	->			List[str]:


          '''simple docstring'''


          snake_case :     str           =   logging.get_logger("diffusers.configuration_utils"				)
          # 30 for warning
          logger.setLevel(30				)

          with CaptureLogger(snake_case__				) as cap_logger:
                 snake_case :     Any           =   DPMSolverMultistepScheduler.from_pretrained(
                     "hf-internal-testing/tiny-stable-diffusion-torch"					,		subfolder="scheduler"				)

          assert dpm.__class__ == DPMSolverMultistepScheduler
          # no warning should be thrown
          assert cap_logger.out == ""



   def        _SCREAMING_SNAKE_CASE     (self	:      Union[str, Any]				)	->			Optional[Any]:


          '''simple docstring'''


          snake_case :     Tuple           =   SampleObject()

          snake_case :     Any           =   {k: v for k, v in config.config.items() if not k.startswith("_"				)}

          # make sure that default config has all keys in `_use_default_values`
          assert set(config_dict.keys()				) == set(config.config._use_default_values				)

          with tempfile.TemporaryDirectory() as tmpdirname:
                 config.save_config(snake_case__				)

                 # now loading it with SampleObject2 should put f into `_use_default_values`
                 snake_case :     Union[str, Any]           =   SampleObjecta.from_config(snake_case__				)

                 assert "f" in config._use_default_values
                 assert config.f == [1, 3]

          # now loading the config, should **NOT** use [1, 3] for `f`, but the default [1, 4] value
          # **BECAUSE** it is part of `config._use_default_values`
          snake_case :     List[Any]           =   SampleObjecta.from_config(config.config				)
          assert new_config.f == [5, 4]

          config.config._use_default_values.pop()
          snake_case :     List[Any]           =   SampleObjecta.from_config(config.config				)
          assert new_config_a.f == [1, 3]

          # Nevertheless "e" should still be correctly loaded to [1, 3] from SampleObject2 instead of defaulting to [1, 5]
          assert new_config_a.e == [1, 3]







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						UpperCAmelCase	(		A_ ,unittest.TestCase				):
   A__      :	Dict		  =  PreTrainedTokenizerFast
   A__      :	str		  =  False
   A__      :	List[Any]		  =  True
   A__      :	List[str]		  =  "tokenizer_file"



   def        _SCREAMING_SNAKE_CASE     (self	:      Dict				)	->			Union[str, Any]:


          '''simple docstring'''


          snake_case :     Union[str, Any]           =   False  # because we don't have pretrained_vocab_files_map
          super().setUp()
          snake_case :     Optional[int]           =   True

          snake_case :     List[str]           =   ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
          snake_case :     Optional[Any]           =   "SaulLu/dummy-tokenizer-bytelevel-bpe"

          # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
          snake_case :     int           =   [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

          snake_case :     List[Any]           =   PreTrainedTokenizerFast.from_pretrained(model_paths[0]				)
          tokenizer.save_pretrained(self.tmpdirname				)



   def        _SCREAMING_SNAKE_CASE     (self	:      Dict				)	->			int:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Any				)	->			int:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Any				)	->			Union[str, Any]:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Tuple				)	->			List[str]:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     Optional[int]           =   self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                        try:
                               snake_case :     Any           =   tempfile.mkdtemp()
                               snake_case :     int           =   self.rust_tokenizer_class.from_pretrained(snake_case__					,		**snake_case__				)

                               tokenizer.save_pretrained(self.tmpdirname				)
                               super().test_training_new_tokenizer()
                        finally:
                               # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                               # is restored
                               shutil.rmtree(self.tmpdirname				)
                               snake_case :     int           =   tmpdirname_orig



   def        _SCREAMING_SNAKE_CASE     (self	:      Tuple				)	->			Dict:


          '''simple docstring'''


          snake_case :     Optional[int]           =   self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                        try:
                               snake_case :     Union[str, Any]           =   tempfile.mkdtemp()
                               snake_case :     str           =   self.rust_tokenizer_class.from_pretrained(snake_case__					,		**snake_case__				)

                               tokenizer.save_pretrained(self.tmpdirname				)
                               super().test_training_new_tokenizer_with_special_tokens_change()
                        finally:
                               # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                               # is restored
                               shutil.rmtree(self.tmpdirname				)
                               snake_case :     Dict           =   tmpdirname_orig



   def        _SCREAMING_SNAKE_CASE     (self	:      int				)	->			Union[str, Any]:


          '''simple docstring'''


          snake_case :     int           =   self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name				)

          snake_case :     int           =   ("a" for _ in range(10_00				))
          snake_case :     Optional[int]           =   tokenizer.train_new_from_iterator(text_iterator=snake_case__					,		length=10_00					,		vocab_size=50				)

          snake_case :     str           =   new_tokenizer.encode("a🤗"				)
          self.assertEqual(snake_case__					,		[64, 1_72, 2_53, 97, 2_45]				)






@require_tokenizers
class 						UpperCAmelCase	(		unittest.TestCase				):



   def        _SCREAMING_SNAKE_CASE     (self	:      Union[str, Any]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     List[str]           =   AutoTokenizer.from_pretrained("bert-base-cased"				)
          snake_case :     Union[str, Any]           =   json.loads(tokenizer._tokenizer.to_str()				)
          snake_case :     Tuple           =   len(snake_case__				)

          with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 snake_case :     List[str]           =   ["tokenizer.4.0.0.json"]
                 tokenizer.save_pretrained(snake_case__				)
                 json.dump(snake_case__					,		open(os.path.join(snake_case__					,		"tokenizer.4.0.0.json"				)					,		"w"				)				)

                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 snake_case :     Optional[Any]           =   AutoTokenizer.from_pretrained(snake_case__				)
                 self.assertEqual(len(snake_case__				)					,		len(snake_case__				) + 1				)
                 snake_case :     Union[str, Any]           =   json.loads(new_tokenizer._tokenizer.to_str()				)
                 self.assertIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)

                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(snake_case__					,		"tokenizer.4.0.0.json"				)					,		os.path.join(snake_case__					,		"tokenizer.42.0.0.json"				)				)
                 snake_case :     Optional[Any]           =   ["tokenizer.42.0.0.json"]
                 tokenizer.save_pretrained(snake_case__				)
                 snake_case :     Dict           =   AutoTokenizer.from_pretrained(snake_case__				)
                 self.assertEqual(len(snake_case__				)					,		len(snake_case__				)				)
                 snake_case :     Optional[int]           =   json.loads(new_tokenizer._tokenizer.to_str()				)
                 self.assertNotIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     int           =   "hf-internal-testing/test-two-tokenizers"

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          snake_case :     int           =   AutoTokenizer.from_pretrained(snake_case__				)
          self.assertEqual(len(snake_case__				)					,		2_89_97				)
          snake_case :     Optional[int]           =   json.loads(tokenizer._tokenizer.to_str()				)
          self.assertIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)

          # Testing an older version by monkey-patching the version in the module it's used.
          import transformers as old_transformers

          snake_case :     Dict           =   "3.0.0"
          snake_case :     Union[str, Any]           =   old_transformers.models.auto.AutoTokenizer.from_pretrained(snake_case__				)
          self.assertEqual(len(snake_case__				)					,		2_89_96				)
          snake_case :     Optional[Any]           =   json.loads(old_tokenizer._tokenizer.to_str()				)
          self.assertNotIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)






@require_tokenizers
class 						UpperCAmelCase	(		unittest.TestCase				):



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			List[Any]:


          '''simple docstring'''


          snake_case :     Tuple           =   PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"				)
          snake_case :     List[Any]           =   "The Matrix is a 1999 science fiction action film."

          with concurrent.futures.ThreadPoolExecutor() as executor:
                 snake_case :     Union[str, Any]           =   [executor.submit(self.fetch					,		snake_case__					,		snake_case__				) for i in range(10				)]
                 snake_case :     Optional[Any]           =   [future.result() for future in futures]
                 self.assertEqual(snake_case__					,		[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10				)]				)



   def        _SCREAMING_SNAKE_CASE     (self	:      str					,		snake_case__	:      List[str]					,		snake_case__	:      str				)	->			List[str]:


          '''simple docstring'''


          return tokenizer.encode(snake_case__					,		truncation="longest_first"					,		padding="longest"				)





