code: 
  
  
  
  
  
  
  """simple docstring"""
  
  import gc
  import os
  import tempfile
  import unittest
  
  import torch
  import torch.nn as nn
  import torch.nn.functional as F
  from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer
  
  from diffusers import AutoencoderKL, DDIMScheduler, StableDiffusionPipeline, UNetaDConditionModel
  from diffusers.loaders import AttnProcsLayers, LoraLoaderMixin
  from diffusers.models.attention_processor import (
      Attention,
      AttnProcessor,
      AttnProcessor2_0,
      LoRAAttnProcessor,
      LoRAAttnProcessor2_0,
      LoRAXFormersAttnProcessor,
      XFormersAttnProcessor,
  )
  from diffusers.utils import TEXT_ENCODER_ATTN_MODULE, floats_tensor, torch_device
  
  def 	__lowerCamelCase			(						__UpperCamelCase )		->					Optional[int]:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		lowerCAmelCase_     :							int             =    {}
  		for name in unet.attn_processors.keys():
  				lowerCAmelCase_     :							List[Any]             =    None if name.endswith("attn1.processor" ) else unet.config.cross_attention_dim
  				if name.startswith("mid_block" ):
  						lowerCAmelCase_     :							Union[str, Any]             =    unet.config.block_out_channels[-1]
  				elif name.startswith("up_blocks" ):
  						lowerCAmelCase_     :							Optional[Any]             =    int(name[len("up_blocks." )] )
  						lowerCAmelCase_     :							Union[str, Any]             =    list(reversed(unet.config.block_out_channels ) )[block_id]
  				elif name.startswith("down_blocks" ):
  						lowerCAmelCase_     :							int             =    int(name[len("down_blocks." )] )
  						lowerCAmelCase_     :							List[Any]             =    unet.config.block_out_channels[block_id]
  				lowerCAmelCase_     :							Dict             =    (
  				    LoRAAttnProcessor2_0 if hasattr(__UpperCamelCase   , "scaled_dot_product_attention" ) else LoRAAttnProcessor
  				)
  				lowerCAmelCase_     :							List[str]             =    lora_attn_processor_class(
  				    hidden_size=__UpperCamelCase   , cross_attention_dim=__UpperCamelCase )
  		lowerCAmelCase_     :							str             =    AttnProcsLayers(__UpperCamelCase )
  		return lora_attn_procs, unet_lora_layers
  
  def 	__lowerCamelCase			(						__UpperCamelCase )		->					Optional[Any]:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		lowerCAmelCase_     :							Optional[Any]             =    {}
  		lowerCAmelCase_     :							List[Any]             =    (
  		    LoRAAttnProcessor2_0 if hasattr(__UpperCamelCase   , "scaled_dot_product_attention" ) else LoRAAttnProcessor
  		)
  		for name, module in text_encoder.named_modules():
  				if name.endswith(__UpperCamelCase ):
  						lowerCAmelCase_     :							Union[str, Any]             =    lora_attn_processor_class(
  						    hidden_size=module.out_proj.out_features   , cross_attention_dim=__UpperCamelCase )
  		return text_lora_attn_procs
  
  def 	__lowerCamelCase			(						__UpperCamelCase )		->					Optional[int]:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		lowerCAmelCase_     :							List[Any]             =    create_text_encoder_lora_attn_procs(__UpperCamelCase )
  		lowerCAmelCase_     :							Any             =    AttnProcsLayers(__UpperCamelCase )
  		return text_encoder_lora_layers
  
  def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase=False )		->					int:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		for _, attn_proc in text_lora_attn_procs.items():
  				# set up.weights
  				for layer_name, layer_module in attn_proc.named_modules():
  						if layer_name.endswith("_lora" ):
  								lowerCAmelCase_     :							Optional[int]             =    (
  								    torch.randn_like(layer_module.up.weight )
  								    if randn_weight
  								    else torch.zeros_like(layer_module.up.weight )
  								)
  								lowerCAmelCase_     :							Dict             =    torch.nn.Parameter(__UpperCamelCase )
  
  
  
  class 	__lowerCamelCase       (				unittest.TestCase    ):
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]			):
  				torch.manual_seed(0			)
  				lowerCAmelCase_     :							List[Any]             =    UNetaDConditionModel(
  				    block_out_channels=(32, 64)		,    layers_per_block=2		,    sample_size=32		,    in_channels=4		,    out_channels=4		,    down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")		,    up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")		,    cross_attention_dim=32		,    )
  				lowerCAmelCase_     :							Optional[int]             =    DDIMScheduler(
  				    beta_start=0.00085		,    beta_end=0.012		,    beta_schedule="scaled_linear"		,    clip_sample=a_		,    set_alpha_to_one=a_		,    steps_offset=1		,    )
  				torch.manual_seed(0			)
  				lowerCAmelCase_     :							Optional[Any]             =    AutoencoderKL(
  				    block_out_channels=[32, 64]		,    in_channels=3		,    out_channels=3		,    down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]		,    up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]		,    latent_channels=4		,    )
  				lowerCAmelCase_     :							Tuple             =    CLIPTextConfig(
  				    bos_token_id=0		,    eos_token_id=2		,    hidden_size=32		,    intermediate_size=37		,    layer_norm_eps=1e-0_5		,    num_attention_heads=4		,    num_hidden_layers=5		,    pad_token_id=1		,    vocab_size=10_00		,    )
  				lowerCAmelCase_     :							Optional[int]             =    CLIPTextModel(a_			)
  				lowerCAmelCase_     :							Tuple             =    CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"			)
  
  				lowerCAmelCase_       ,			lowerCAmelCase_     :							Dict             =    create_unet_lora_layers(a_			)
  				lowerCAmelCase_     :							Optional[Any]             =    create_text_encoder_lora_layers(a_			)
  
  				lowerCAmelCase_     :							Optional[int]             =    {
  				    "unet": unet,
  				    "scheduler": scheduler,
  				    "vae": vae,
  				    "text_encoder": text_encoder,
  				    "tokenizer": tokenizer,
  				    "safety_checker": None,
  				    "feature_extractor": None,
  				}
  				lowerCAmelCase_     :							Tuple             =    {
  				    "unet_lora_layers": unet_lora_layers,
  				    "text_encoder_lora_layers": text_encoder_lora_layers,
  				    "unet_lora_attn_procs": unet_lora_attn_procs,
  				}
  				return pipeline_components, lora_components
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Any			):
  				lowerCAmelCase_     :							Optional[int]             =    1
  				lowerCAmelCase_     :							Union[str, Any]             =    10
  				lowerCAmelCase_     :							Dict             =    4
  				lowerCAmelCase_     :							int             =    (32, 32)
  
  				lowerCAmelCase_     :							Union[str, Any]             =    torch.manual_seed(0			)
  				lowerCAmelCase_     :							Dict             =    floats_tensor((batch_size, num_channels) + sizes			)
  				lowerCAmelCase_     :							Any             =    torch.randint(1		,    a_		,    size=(batch_size, sequence_length)		,    generator=a_			)
  
  				lowerCAmelCase_     :							Optional[Any]             =    {
  				    "prompt": "A painting of a squirrel eating a burger",
  				    "generator": generator,
  				    "num_inference_steps": 2,
  				    "guidance_scale": 6.0,
  				    "output_type": "numpy",
  				}
  
  				return noise, input_ids, pipeline_inputs
  
  				# copied from: https://colab.research.google.com/gist/sayakpaul/df2ef6e1ae6d8c10a49d859883b10860/scratchpad.ipynb
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Union[str, Any]			):
  				lowerCAmelCase_     :							str             =    77
  
  				lowerCAmelCase_     :							int             =    torch.randint(2		,    56		,    size=(1, max_seq_length)		,    generator=torch.manual_seed(0			)			)
  
  				lowerCAmelCase_     :							int             =    {}
  				lowerCAmelCase_     :							int             =    inputs
  				return prepared_inputs
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]		,    a_							:			Optional[Any]			):
  				lowerCAmelCase_       ,			lowerCAmelCase_     :							Any             =    self.get_dummy_components()
  				LoraLoaderMixin.save_lora_weights(
  				    save_directory=a_		,    unet_lora_layers=lora_components["unet_lora_layers"]		,    text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]		,    )
  				self.assertTrue(os.path.isfile(os.path.join(a_		,    "pytorch_lora_weights.bin"			)			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_       ,			lowerCAmelCase_     :							int             =    self.get_dummy_components()
  				lowerCAmelCase_     :							Dict             =    StableDiffusionPipeline(**a_			)
  				lowerCAmelCase_     :							Dict             =    sd_pipe.to(a_			)
  				sd_pipe.set_progress_bar_config(disable=a_			)
  
  				lowerCAmelCase_       ,			lowerCAmelCase_       ,			lowerCAmelCase_     :							Dict             =    self.get_dummy_inputs()
  
  				lowerCAmelCase_     :							Tuple             =    sd_pipe(**a_			).images
  				lowerCAmelCase_     :							Tuple             =    original_images[0, -3:, -3:, -1]
  
  				with tempfile.TemporaryDirectory() as tmpdirname:
  						LoraLoaderMixin.save_lora_weights(
  						    save_directory=a_		,    unet_lora_layers=lora_components["unet_lora_layers"]		,    text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]		,    )
  						self.assertTrue(os.path.isfile(os.path.join(a_		,    "pytorch_lora_weights.bin"			)			)			)
  						sd_pipe.load_lora_weights(a_			)
  
  				lowerCAmelCase_     :							Any             =    sd_pipe(**a_			).images
  				lowerCAmelCase_     :							str             =    lora_images[0, -3:, -3:, -1]
  
  				# Outputs shouldn't match.
  				self.assertFalse(torch.allclose(torch.from_numpy(a_			)		,    torch.from_numpy(a_			)			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				lowerCAmelCase_       ,			lowerCAmelCase_     :							Union[str, Any]             =    self.get_dummy_components()
  				lowerCAmelCase_     :							str             =    StableDiffusionPipeline(**a_			)
  				lowerCAmelCase_     :							List[Any]             =    sd_pipe.to(a_			)
  				sd_pipe.set_progress_bar_config(disable=a_			)
  
  				lowerCAmelCase_       ,			lowerCAmelCase_       ,			lowerCAmelCase_     :							Tuple             =    self.get_dummy_inputs()
  
  				lowerCAmelCase_     :							Any             =    sd_pipe(**a_			).images
  				lowerCAmelCase_     :							List[Any]             =    original_images[0, -3:, -3:, -1]
  
  				with tempfile.TemporaryDirectory() as tmpdirname:
  						LoraLoaderMixin.save_lora_weights(
  						    save_directory=a_		,    unet_lora_layers=lora_components["unet_lora_layers"]		,    text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]		,    safe_serialization=a_		,    )
  						self.assertTrue(os.path.isfile(os.path.join(a_		,    "pytorch_lora_weights.safetensors"			)			)			)
  						sd_pipe.load_lora_weights(a_			)
  
  				lowerCAmelCase_     :							Optional[int]             =    sd_pipe(**a_			).images
  				lowerCAmelCase_     :							Tuple             =    lora_images[0, -3:, -3:, -1]
  
  				# Outputs shouldn't match.
  				self.assertFalse(torch.allclose(torch.from_numpy(a_			)		,    torch.from_numpy(a_			)			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_       ,			lowerCAmelCase_     :							str             =    self.get_dummy_components()
  				lowerCAmelCase_     :							Optional[int]             =    lora_components["unet_lora_attn_procs"]
  				lowerCAmelCase_     :							Optional[int]             =    StableDiffusionPipeline(**a_			)
  				lowerCAmelCase_     :							Optional[int]             =    sd_pipe.to(a_			)
  				sd_pipe.set_progress_bar_config(disable=a_			)
  
  				lowerCAmelCase_       ,			lowerCAmelCase_       ,			lowerCAmelCase_     :							str             =    self.get_dummy_inputs()
  
  				lowerCAmelCase_     :							Optional[Any]             =    sd_pipe(**a_			).images
  				lowerCAmelCase_     :							Union[str, Any]             =    original_images[0, -3:, -3:, -1]
  
  				with tempfile.TemporaryDirectory() as tmpdirname:
  						lowerCAmelCase_     :							List[Any]             =    sd_pipe.unet
  						unet.set_attn_processor(a_			)
  						unet.save_attn_procs(a_			)
  						self.assertTrue(os.path.isfile(os.path.join(a_		,    "pytorch_lora_weights.bin"			)			)			)
  						sd_pipe.load_lora_weights(a_			)
  
  				lowerCAmelCase_     :							Any             =    sd_pipe(**a_			).images
  				lowerCAmelCase_     :							Optional[Any]             =    lora_images[0, -3:, -3:, -1]
  
  				# Outputs shouldn't match.
  				self.assertFalse(torch.allclose(torch.from_numpy(a_			)		,    torch.from_numpy(a_			)			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[str]			):
  				lowerCAmelCase_       ,			lowerCAmelCase_     :							Optional[Any]             =    self.get_dummy_components()
  				lowerCAmelCase_     :							Optional[Any]             =    StableDiffusionPipeline(**a_			)
  
  				lowerCAmelCase_     :							str             =    self.get_dummy_tokens()
  
  				# inference without lora
  				lowerCAmelCase_     :							str             =    pipe.text_encoder(**a_			)[0]
  				assert outputs_without_lora.shape == (1, 77, 32)
  
  				# create lora_attn_procs with zeroed out up.weights
  				lowerCAmelCase_     :							Any             =    create_text_encoder_lora_attn_procs(pipe.text_encoder			)
  				set_lora_up_weights(a_		,    randn_weight=a_			)
  
  				# monkey patch
  				pipe._modify_text_encoder(a_			)
  
  				# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
  				del text_attn_procs
  				gc.collect()
  
  				# inference with lora
  				lowerCAmelCase_     :							List[str]             =    pipe.text_encoder(**a_			)[0]
  				assert outputs_with_lora.shape == (1, 77, 32)
  
  				assert torch.allclose(
  				    a_		,    a_			), "lora_up_weight are all zero, so the lora outputs should be the same to without lora outputs"
  
  				# create lora_attn_procs with randn up.weights
  				lowerCAmelCase_     :							Union[str, Any]             =    create_text_encoder_lora_attn_procs(pipe.text_encoder			)
  				set_lora_up_weights(a_		,    randn_weight=a_			)
  
  				# monkey patch
  				pipe._modify_text_encoder(a_			)
  
  				# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
  				del text_attn_procs
  				gc.collect()
  
  				# inference with lora
  				lowerCAmelCase_     :							Dict             =    pipe.text_encoder(**a_			)[0]
  				assert outputs_with_lora.shape == (1, 77, 32)
  
  				assert not torch.allclose(
  				    a_		,    a_			), "lora_up_weight are not zero, so the lora outputs should be different to without lora outputs"
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_       ,			lowerCAmelCase_     :							Union[str, Any]             =    self.get_dummy_components()
  				lowerCAmelCase_     :							Optional[Any]             =    StableDiffusionPipeline(**a_			)
  
  				lowerCAmelCase_     :							Optional[int]             =    self.get_dummy_tokens()
  
  				# inference without lora
  				lowerCAmelCase_     :							List[str]             =    pipe.text_encoder(**a_			)[0]
  				assert outputs_without_lora.shape == (1, 77, 32)
  
  				# create lora_attn_procs with randn up.weights
  				lowerCAmelCase_     :							List[Any]             =    create_text_encoder_lora_attn_procs(pipe.text_encoder			)
  				set_lora_up_weights(a_		,    randn_weight=a_			)
  
  				# monkey patch
  				pipe._modify_text_encoder(a_			)
  
  				# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
  				del text_attn_procs
  				gc.collect()
  
  				# inference with lora
  				lowerCAmelCase_     :							Optional[int]             =    pipe.text_encoder(**a_			)[0]
  				assert outputs_with_lora.shape == (1, 77, 32)
  
  				assert not torch.allclose(
  				    a_		,    a_			), "lora outputs should be different to without lora outputs"
  
  				# remove monkey patch
  				pipe._remove_text_encoder_monkey_patch()
  
  				# inference with removed lora
  				lowerCAmelCase_     :							List[str]             =    pipe.text_encoder(**a_			)[0]
  				assert outputs_without_lora_removed.shape == (1, 77, 32)
  
  				assert torch.allclose(
  				    a_		,    a_			), "remove lora monkey patch should restore the original outputs"
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_       ,			lowerCAmelCase_     :							Dict             =    self.get_dummy_components()
  				lowerCAmelCase_     :							Optional[Any]             =    StableDiffusionPipeline(**a_			)
  				lowerCAmelCase_     :							List[Any]             =    sd_pipe.to(a_			)
  				sd_pipe.set_progress_bar_config(disable=a_			)
  
  				lowerCAmelCase_       ,			lowerCAmelCase_       ,			lowerCAmelCase_     :							List[Any]             =    self.get_dummy_inputs()
  
  				with tempfile.TemporaryDirectory() as tmpdirname:
  						LoraLoaderMixin.save_lora_weights(
  						    save_directory=a_		,    unet_lora_layers=lora_components["unet_lora_layers"]		,    text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]		,    )
  						self.assertTrue(os.path.isfile(os.path.join(a_		,    "pytorch_lora_weights.bin"			)			)			)
  						sd_pipe.load_lora_weights(a_			)
  
  				lowerCAmelCase_     :							Dict             =    sd_pipe(**a_			).images
  				lowerCAmelCase_     :							Optional[int]             =    lora_images[0, -3:, -3:, -1]
  
  				lowerCAmelCase_     :							int             =    sd_pipe(**a_		,    cross_attention_kwargs={"scale": 0.5}			).images
  				lowerCAmelCase_     :							List[str]             =    lora_images_with_scale[0, -3:, -3:, -1]
  
  				# Outputs shouldn't match.
  				self.assertFalse(
  				    torch.allclose(torch.from_numpy(a_			)		,    torch.from_numpy(a_			)			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Union[str, Any]			):
  				with tempfile.TemporaryDirectory() as tmpdirname:
  						self.create_lora_weight_file(a_			)
  
  						lowerCAmelCase_       ,			lowerCAmelCase_     :							Any             =    self.get_dummy_components()
  						lowerCAmelCase_     :							int             =    StableDiffusionPipeline(**a_			)
  						lowerCAmelCase_     :							List[str]             =    sd_pipe.to(a_			)
  						sd_pipe.set_progress_bar_config(disable=a_			)
  
  						# check if vanilla attention processors are used
  						for _, module in sd_pipe.unet.named_modules():
  								if isinstance(a_		,    a_			):
  										self.assertIsInstance(module.processor		,    (AttnProcessor, AttnProcessor2_0)			)
  
              # load LoRA weight file
  						sd_pipe.load_lora_weights(a_			)
  
  						# check if lora attention processors are used
  						for _, module in sd_pipe.unet.named_modules():
  								if isinstance(a_		,    a_			):
  										lowerCAmelCase_     :							List[str]             =    (
  										    LoRAAttnProcessor2_0 if hasattr(a_		,    "scaled_dot_product_attention"			) else LoRAAttnProcessor
  										)
  										self.assertIsInstance(module.processor		,    a_			)
  
  
  
  
  
  		@unittest.skipIf(torch_device != "cuda"		,    "This test is supposed to run on GPU"			)
  		def      lowerCamelCase (     self							:			int			):
  				with tempfile.TemporaryDirectory() as tmpdirname:
  						self.create_lora_weight_file(a_			)
  
  						lowerCAmelCase_       ,			lowerCAmelCase_     :							Union[str, Any]             =    self.get_dummy_components()
  						lowerCAmelCase_     :							Any             =    StableDiffusionPipeline(**a_			)
  						lowerCAmelCase_     :							Union[str, Any]             =    sd_pipe.to(a_			)
  						sd_pipe.set_progress_bar_config(disable=a_			)
  
  						# enable XFormers
  						sd_pipe.enable_xformers_memory_efficient_attention()
  
  						# check if xFormers attention processors are used
  						for _, module in sd_pipe.unet.named_modules():
  								if isinstance(a_		,    a_			):
  										self.assertIsInstance(module.processor		,    a_			)
  
              # load LoRA weight file
  						sd_pipe.load_lora_weights(a_			)
  
  						# check if lora attention processors are used
  						for _, module in sd_pipe.unet.named_modules():
  								if isinstance(a_		,    a_			):
  										self.assertIsInstance(module.processor		,    a_			)
  
  
  
  
  
  		@unittest.skipIf(torch_device != "cuda"		,    "This test is supposed to run on GPU"			)
  		def      lowerCamelCase (     self							:			Any			):
  				lowerCAmelCase_       ,			lowerCAmelCase_     :							Tuple             =    self.get_dummy_components()
  				lowerCAmelCase_     :							Union[str, Any]             =    StableDiffusionPipeline(**a_			)
  				lowerCAmelCase_     :							int             =    sd_pipe.to(a_			)
  				sd_pipe.set_progress_bar_config(disable=a_			)
  
  				lowerCAmelCase_       ,			lowerCAmelCase_       ,			lowerCAmelCase_     :							Optional[int]             =    self.get_dummy_inputs()
  
  				# enable XFormers
  				sd_pipe.enable_xformers_memory_efficient_attention()
  
  				lowerCAmelCase_     :							List[Any]             =    sd_pipe(**a_			).images
  				lowerCAmelCase_     :							List[Any]             =    original_images[0, -3:, -3:, -1]
  
  				with tempfile.TemporaryDirectory() as tmpdirname:
  						LoraLoaderMixin.save_lora_weights(
  						    save_directory=a_		,    unet_lora_layers=lora_components["unet_lora_layers"]		,    text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]		,    )
  						self.assertTrue(os.path.isfile(os.path.join(a_		,    "pytorch_lora_weights.bin"			)			)			)
  						sd_pipe.load_lora_weights(a_			)
  
  				lowerCAmelCase_     :							str             =    sd_pipe(**a_			).images
  				lowerCAmelCase_     :							Dict             =    lora_images[0, -3:, -3:, -1]
  
  				# Outputs shouldn't match.
  				self.assertFalse(torch.allclose(torch.from_numpy(a_			)		,    torch.from_numpy(a_			)			)			)
  
style_context: 
  
  
  
  
  
  
  """simple docstring"""
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  @require_tokenizers
  class 	__lowerCamelCase       (				A__				,							unittest.TestCase    ):
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  		a_			:       List[str]      =							RealmTokenizer
  		a_			:       int      =							RealmTokenizerFast
  		a_			:       str      =							True
  		a_			:       Tuple      =							True
  		a_			:       List[Any]      =							filter_non_english
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				super().setUp()
  
  				lowerCAmelCase_     :							int             =    [
  				    "[UNK]",
  				    "[CLS]",
  				    "[SEP]",
  				    "[PAD]",
  				    "[MASK]",
  				    "want",
  				    "##want",
  				    "##ed",
  				    "wa",
  				    "un",
  				    "runn",
  				    "##ing",
  				    ",",
  				    "low",
  				    "lowest",
  				]
  				lowerCAmelCase_     :							List[Any]             =    os.path.join(self.tmpdirname		,    VOCAB_FILES_NAMES["vocab_file"]			)
  				with open(self.vocab_file		,    "w"		,    encoding="utf-8"			) as vocab_writer:
  						vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]		,    a_							:			Any			):
  				lowerCAmelCase_     :							int             =    "UNwant\u00E9d,running"
  				lowerCAmelCase_     :							Optional[Any]             =    "unwanted, running"
  				return input_text, output_text
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_     :							Tuple             =    self.tokenizer_class(self.vocab_file			)
  
  				lowerCAmelCase_     :							List[str]             =    tokenizer.tokenize("UNwant\u00E9d,running"			)
  				self.assertListEqual(a_		,    ["un", "##want", "##ed", ",", "runn", "##ing"]			)
  				self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)		,    [9, 6, 7, 12, 10, 11]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Dict			):
  				if not self.test_rust_tokenizer:
  						return
  
  				lowerCAmelCase_     :							str             =    self.get_tokenizer()
  				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer()
  
  				lowerCAmelCase_     :							Optional[Any]             =    "UNwant\u00E9d,running"
  
  				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.tokenize(a_			)
  				lowerCAmelCase_     :							List[Any]             =    rust_tokenizer.tokenize(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							str             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							int             =    self.get_rust_tokenizer()
  				lowerCAmelCase_     :							Any             =    tokenizer.encode(a_			)
  				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.encode(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				# With lower casing
  				lowerCAmelCase_     :							Dict             =    self.get_tokenizer(do_lower_case=a_			)
  				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer(do_lower_case=a_			)
  
  				lowerCAmelCase_     :							Dict             =    "UNwant\u00E9d,running"
  
  				lowerCAmelCase_     :							Union[str, Any]             =    tokenizer.tokenize(a_			)
  				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.tokenize(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
  				lowerCAmelCase_     :							str             =    tokenizer.encode(a_			)
  				lowerCAmelCase_     :							Optional[int]             =    rust_tokenizer.encode(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]			):
  				lowerCAmelCase_     :							Any             =    BasicTokenizer()
  
  				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)		,    ["ah", "\u535A", "\u63A8", "zz"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["hello", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				lowerCAmelCase_     :							Optional[int]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hällo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["h\u00E9llo"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[Any]			):
  				lowerCAmelCase_     :							Optional[Any]             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[Any]			):
  				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HäLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Dict			):
  				lowerCAmelCase_     :							List[str]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HaLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[str]			):
  				lowerCAmelCase_     :							Any             =    BasicTokenizer(do_lower_case=a_		,    never_split=["[UNK]"]			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				lowerCAmelCase_     :							int             =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
  				lowerCAmelCase_     :							Union[str, Any]             =    {}
  				for i, token in enumerate(a_			):
  						lowerCAmelCase_     :							Any             =    i
  				lowerCAmelCase_     :							int             =    WordpieceTokenizer(vocab=a_		,    unk_token="[UNK]"			)
  
  				self.assertListEqual(tokenizer.tokenize(""			)		,    []			)
  
  				self.assertListEqual(tokenizer.tokenize("unwanted running"			)		,    ["un", "##want", "##ed", "runn", "##ing"]			)
  
  				self.assertListEqual(tokenizer.tokenize("unwantedX running"			)		,    ["[UNK]", "runn", "##ing"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				self.assertTrue(_is_whitespace(" "			)			)
  				self.assertTrue(_is_whitespace("\t"			)			)
  				self.assertTrue(_is_whitespace("\r"			)			)
  				self.assertTrue(_is_whitespace("\n"			)			)
  				self.assertTrue(_is_whitespace("\u00A0"			)			)
  
  				self.assertFalse(_is_whitespace("A"			)			)
  				self.assertFalse(_is_whitespace("-"			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				self.assertTrue(_is_control("\u0005"			)			)
  
  				self.assertFalse(_is_control("A"			)			)
  				self.assertFalse(_is_control(" "			)			)
  				self.assertFalse(_is_control("\t"			)			)
  				self.assertFalse(_is_control("\r"			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				self.assertTrue(_is_punctuation("-"			)			)
  				self.assertTrue(_is_punctuation("$"			)			)
  				self.assertTrue(_is_punctuation("`"			)			)
  				self.assertTrue(_is_punctuation("."			)			)
  
  				self.assertFalse(_is_punctuation("A"			)			)
  				self.assertFalse(_is_punctuation(" "			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				lowerCAmelCase_     :							Any             =    self.get_tokenizer()
  
  				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  				self.assertListEqual([tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)
  
  				if self.test_rust_tokenizer:
  						lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
  						self.assertListEqual(
  						    [rust_tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)
  
  
  
  
  
  		@slow
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_     :							int             =    self.tokenizer_class.from_pretrained("bert-base-uncased"			)
  
  				lowerCAmelCase_     :							List[Any]             =    tokenizer.encode("sequence builders"		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Optional[int]             =    tokenizer.encode("multi-sequence build"		,    add_special_tokens=a_			)
  
  				lowerCAmelCase_     :							Dict             =    tokenizer.build_inputs_with_special_tokens(a_			)
  				lowerCAmelCase_     :							Tuple             =    tokenizer.build_inputs_with_special_tokens(a_		,    a_			)
  
  				assert encoded_sentence == [1_01] + text + [1_02]
  				assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]			):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
  								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)
  
  								lowerCAmelCase_     :							Tuple             =    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  								lowerCAmelCase_     :							List[str]             =    tokenizer_r.encode_plus(
  								    a_		,    return_attention_mask=a_		,    return_token_type_ids=a_		,    return_offsets_mapping=a_		,    add_special_tokens=a_		,    )
  
  								lowerCAmelCase_     :							List[str]             =    tokenizer_r.do_lower_case if hasattr(a_		,    "do_lower_case"			) else False
  								lowerCAmelCase_     :							int             =    (
  								    [
  								        ((0, 0), tokenizer_r.cls_token),
  								        ((0, 1), "A"),
  								        ((1, 2), ","),
  								        ((3, 5), "na"),
  								        ((5, 6), "##ï"),
  								        ((6, 8), "##ve"),
  								        ((9, 15), tokenizer_r.mask_token),
  								        ((16, 21), "Allen"),
  								        ((21, 23), "##NL"),
  								        ((23, 24), "##P"),
  								        ((25, 33), "sentence"),
  								        ((33, 34), "."),
  								        ((0, 0), tokenizer_r.sep_token),
  								    ]
  								    if not do_lower_case
  								    else [
  								        ((0, 0), tokenizer_r.cls_token),
  								        ((0, 1), "a"),
  								        ((1, 2), ","),
  								        ((3, 8), "naive"),
  								        ((9, 15), tokenizer_r.mask_token),
  								        ((16, 21), "allen"),
  								        ((21, 23), "##nl"),
  								        ((23, 24), "##p"),
  								        ((25, 33), "sentence"),
  								        ((33, 34), "."),
  								        ((0, 0), tokenizer_r.sep_token),
  								    ]
  								)
  
  								self.assertEqual(
  								    [e[1] for e in expected_results]		,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
  								self.assertEqual([e[0] for e in expected_results]		,    tokens["offset_mapping"]			)
  
  
  
  
  
  		@slow
  		def      lowerCamelCase (     self							:			int			):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
  								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)
  								lowerCAmelCase_     :							Tuple             =    self.tokenizer_class.from_pretrained(a_		,    **a_			)
  								lowerCAmelCase_     :							Dict             =    [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]
  
  								lowerCAmelCase_     :							Optional[Any]             =    tokenizer_r.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)
  								lowerCAmelCase_     :							int             =    tokenizer_p.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)
  
  								lowerCAmelCase_     :							Dict             =    (2, 2, 10)
  
  								self.assertEqual(encoded_sentence_r["input_ids"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_r["attention_mask"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_r["token_type_ids"].shape		,    a_			)
  
  								self.assertEqual(encoded_sentence_p["input_ids"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_p["attention_mask"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_p["token_type_ids"].shape		,    a_			)
  