



'''simple docstring'''





from typing import Callable, List, Optional, Tuple, Union

import torch
from transformers import CLIPTextModel, CLIPTokenizer

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin, TransformeraDModel, VQModel
from ...schedulers import VQDiffusionScheduler
from ...utils import logging
from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput


_lowerCAmelCase      						=	logging.get_logger(__name__)  # pylint: disable=invalid-name






class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_						, SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  @register_to_config
  def __init__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None  ,__UpperCAmelCase = None      )    ->		Dict:
      super().__init__()

      lowerCAmelCase__ :  List[Any]							      =					learnable

      if self.learnable:
          assert hidden_size is not None, "learnable=True requires `hidden_size` to be set"
          assert length is not None, "learnable=True requires `length` to be set"

          lowerCAmelCase__ :  Dict							      =					torch.zeros(__UpperCAmelCase  ,__UpperCAmelCase      )
      else:
          lowerCAmelCase__ :  Optional[int]							      =					None

      lowerCAmelCase__ :  Union[str, Any]							      =					torch.nn.Parameter(__UpperCAmelCase      )







class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  __lowercase							:      VQModel
  __lowercase							:      CLIPTextModel
  __lowercase							:      CLIPTokenizer
  __lowercase							:      TransformeraDModel
  __lowercase							:      LearnedClassifierFreeSamplingEmbeddings
  __lowercase							:      VQDiffusionScheduler




  def __init__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,)    ->		Union[str, Any]:
      super().__init__()

      self.register_modules(
          vqvae=__UpperCAmelCase  ,transformer=__UpperCAmelCase  ,text_encoder=__UpperCAmelCase  ,tokenizer=__UpperCAmelCase  ,scheduler=__UpperCAmelCase  ,learned_classifier_free_sampling_embeddings=__UpperCAmelCase  ,)




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Optional[int]:
      lowerCAmelCase__ :  Optional[int]							      =					len(__UpperCAmelCase      ) if isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ) else 1

      # get prompt text embeddings
      lowerCAmelCase__ :  int							      =					self.tokenizer(
          __UpperCAmelCase  ,padding="""max_length"""  ,max_length=self.tokenizer.model_max_length  ,return_tensors="""pt"""  ,)
      lowerCAmelCase__ :  Dict							      =					text_inputs.input_ids

      if text_input_ids.shape[-1] > self.tokenizer.model_max_length:
          lowerCAmelCase__ :  List[Any]							      =					self.tokenizer.batch_decode(text_input_ids[:, self.tokenizer.model_max_length :]      )
          logger.warning(
              """The following part of your input was truncated because CLIP can only handle sequences up to"""
              F""" {self.tokenizer.model_max_length} tokens: {removed_text}"""      )
          lowerCAmelCase__ :  Optional[Any]							      =					text_input_ids[:, : self.tokenizer.model_max_length]
      lowerCAmelCase__ :  Union[str, Any]							      =					self.text_encoder(text_input_ids.to(self.device      )      )[0]

      # NOTE: This additional step of normalizing the text embeddings is from VQ-Diffusion.
      # While CLIP does normalize the pooled output of the text transformer when combining
      # the image and text embeddings, CLIP does not directly normalize the last hidden state.
      #
      # CLIP normalizing the pooled output.
      # https://github.com/huggingface/transformers/blob/d92e22d1f28324f513f3080e5c47c071a3916721/src/transformers/models/clip/modeling_clip.py#L1052-L1053
      lowerCAmelCase__ :  List[str]							      =					prompt_embeds / prompt_embeds.norm(dim=-1  ,keepdim=__UpperCAmelCase      )

      # duplicate text embeddings for each generation per prompt
      lowerCAmelCase__ :  Any							      =					prompt_embeds.repeat_interleave(__UpperCAmelCase  ,dim=0      )

      if do_classifier_free_guidance:
          if self.learned_classifier_free_sampling_embeddings.learnable:
              lowerCAmelCase__ :  Dict							      =					self.learned_classifier_free_sampling_embeddings.embeddings
              lowerCAmelCase__ :  Optional[Any]							      =					negative_prompt_embeds.unsqueeze(0      ).repeat(__UpperCAmelCase  ,1  ,1      )
          else:
              lowerCAmelCase__ :  Optional[Any]							      =					[""""""] * batch_size

              lowerCAmelCase__ :  int							      =					text_input_ids.shape[-1]
              lowerCAmelCase__ :  str							      =					self.tokenizer(
                  __UpperCAmelCase  ,padding="""max_length"""  ,max_length=__UpperCAmelCase  ,truncation=__UpperCAmelCase  ,return_tensors="""pt"""  ,)
              lowerCAmelCase__ :  Union[str, Any]							      =					self.text_encoder(uncond_input.input_ids.to(self.device      )      )[0]
              # See comment for normalizing text embeddings
              lowerCAmelCase__ :  Union[str, Any]							      =					negative_prompt_embeds / negative_prompt_embeds.norm(dim=-1  ,keepdim=__UpperCAmelCase      )

          # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
          lowerCAmelCase__ :  Optional[int]							      =					negative_prompt_embeds.shape[1]
          lowerCAmelCase__ :  int							      =					negative_prompt_embeds.repeat(1  ,__UpperCAmelCase  ,1      )
          lowerCAmelCase__ :  Tuple							      =					negative_prompt_embeds.view(batch_size * num_images_per_prompt  ,__UpperCAmelCase  ,-1      )

          # For classifier free guidance, we need to do two forward passes.
          # Here we concatenate the unconditional and text embeddings into a single batch
          # to avoid doing two forward passes
          lowerCAmelCase__ :  List[Any]							      =					torch.cat([negative_prompt_embeds, prompt_embeds]      )

      return prompt_embeds




  @torch.no_grad()
  def __call__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = 100  ,__UpperCAmelCase = 5.0  ,__UpperCAmelCase = 1.0  ,__UpperCAmelCase = 1  ,__UpperCAmelCase = None  ,__UpperCAmelCase = None  ,__UpperCAmelCase = "pil"  ,__UpperCAmelCase = True  ,__UpperCAmelCase = None  ,__UpperCAmelCase = 1  ,)    ->		Union[ImagePipelineOutput, Tuple]:
      if isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ):
          lowerCAmelCase__ :  Tuple							      =					1
      elif isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ):
          lowerCAmelCase__ :  int							      =					len(__UpperCAmelCase      )
      else:
          raise ValueError(F"""`prompt` has to be of type `str` or `list` but is {type(__UpperCAmelCase      )}"""      )

      lowerCAmelCase__ :  Optional[Any]							      =					batch_size * num_images_per_prompt

      lowerCAmelCase__ :  Union[str, Any]							      =					guidance_scale > 1.0

      lowerCAmelCase__ :  Any							      =					self._encode_prompt(__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )

      if (callback_steps is None) or (
          callback_steps is not None and (not isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ) or callback_steps <= 0)
      ):
          raise ValueError(
              F"""`callback_steps` has to be a positive integer but is {callback_steps} of type"""
              F""" {type(__UpperCAmelCase      )}."""      )

      # get the initial completely masked latents unless the user supplied it

      lowerCAmelCase__ :  Dict							      =					(batch_size, self.transformer.num_latent_pixels)
      if latents is None:
          lowerCAmelCase__ :  List[Any]							      =					self.transformer.num_vector_embeds - 1
          lowerCAmelCase__ :  int							      =					torch.full(__UpperCAmelCase  ,__UpperCAmelCase      ).to(self.device      )
      else:
          if latents.shape != latents_shape:
              raise ValueError(F"""Unexpected latents shape, got {latents.shape}, expected {latents_shape}"""      )
          if (latents < 0).any() or (latents >= self.transformer.num_vector_embeds).any():
              raise ValueError(
                  """Unexpected latents value(s). All latents be valid embedding indices i.e. in the range 0,"""
                  F""" {self.transformer.num_vector_embeds - 1} (inclusive)."""      )
          lowerCAmelCase__ :  List[str]							      =					latents.to(self.device      )

      # set timesteps
      self.scheduler.set_timesteps(__UpperCAmelCase  ,device=self.device      )

      lowerCAmelCase__ :  Any							      =					self.scheduler.timesteps.to(self.device      )

      lowerCAmelCase__ :  int							      =					latents

      for i, t in enumerate(self.progress_bar(__UpperCAmelCase      )      ):
          # expand the sample if we are doing classifier free guidance
          lowerCAmelCase__ :  Union[str, Any]							      =					torch.cat([sample] * 2      ) if do_classifier_free_guidance else sample

          # predict the un-noised image
          # model_output == `log_p_x_0`
          lowerCAmelCase__ :  List[str]							      =					self.transformer(__UpperCAmelCase  ,encoder_hidden_states=__UpperCAmelCase  ,timestep=__UpperCAmelCase      ).sample

          if do_classifier_free_guidance:
              lowerCAmelCase__      ,   lowerCAmelCase__ :  List[str]							      =					model_output.chunk(2      )
              lowerCAmelCase__ :  List[Any]							      =					model_output_uncond + guidance_scale * (model_output_text - model_output_uncond)
              model_output -= torch.logsumexp(__UpperCAmelCase  ,dim=1  ,keepdim=__UpperCAmelCase      )

          lowerCAmelCase__ :  List[str]							      =					self.truncate(__UpperCAmelCase  ,__UpperCAmelCase      )

          # remove `log(0)`'s (`-inf`s)
          lowerCAmelCase__ :  Optional[Any]							      =					model_output.clamp(-70      )

          # compute the previous noisy sample x_t -> x_t-1
          lowerCAmelCase__ :  List[str]							      =					self.scheduler.step(__UpperCAmelCase  ,timestep=__UpperCAmelCase  ,sample=__UpperCAmelCase  ,generator=__UpperCAmelCase      ).prev_sample

          # call the callback, if provided
          if callback is not None and i % callback_steps == 0:
              callback(__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  Tuple							      =					self.vqvae.config.vq_embed_dim
      lowerCAmelCase__ :  Optional[int]							      =					(batch_size, self.transformer.height, self.transformer.width, embedding_channels)
      lowerCAmelCase__ :  List[str]							      =					self.vqvae.quantize.get_codebook_entry(__UpperCAmelCase  ,shape=__UpperCAmelCase      )
      lowerCAmelCase__ :  int							      =					self.vqvae.decode(__UpperCAmelCase  ,force_not_quantize=__UpperCAmelCase      ).sample

      lowerCAmelCase__ :  Dict							      =					(image / 2 + 0.5).clamp(0  ,1      )
      lowerCAmelCase__ :  Union[str, Any]							      =					image.cpu().permute(0  ,2  ,3  ,1      ).numpy()

      if output_type == "pil":
          lowerCAmelCase__ :  Dict							      =					self.numpy_to_pil(__UpperCAmelCase      )

      if not return_dict:
          return (image,)

      return ImagePipelineOutput(images=__UpperCAmelCase      )






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		torch.FloatTensor:
      lowerCAmelCase__      ,   lowerCAmelCase__ :  Any							      =					torch.sort(__UpperCAmelCase  ,1  ,descending=__UpperCAmelCase      )
      lowerCAmelCase__ :  Any							      =					torch.exp(__UpperCAmelCase      )
      lowerCAmelCase__ :  Dict							      =					sorted_p_x_0.cumsum(dim=1      ) < truncation_rate

      # Ensure that at least the largest probability is not zeroed out
      lowerCAmelCase__ :  Dict							      =					torch.full_like(keep_mask[:, 0:1, :]  ,__UpperCAmelCase      )
      lowerCAmelCase__ :  Optional[int]							      =					torch.cat((all_true, keep_mask)  ,dim=1      )
      lowerCAmelCase__ :  str							      =					keep_mask[:, :-1, :]

      lowerCAmelCase__ :  Optional[Any]							      =					keep_mask.gather(1  ,indices.argsort(1      )      )

      lowerCAmelCase__ :  List[Any]							      =					log_p_x_0.clone()

      lowerCAmelCase__ :  Optional[Any]							      =					-torch.inf  # -inf = log(0)

      return rv







import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class       a							(			__lowerCamelCase  ,							unittest.TestCase				):
		__lowerCAmelCase	:    int											=					BertTokenizer
		__lowerCAmelCase	:    int											=					BertTokenizerFast
		__lowerCAmelCase	:    int											=					True
		__lowerCAmelCase	:    List[Any]											=					True
		__lowerCAmelCase	:    Any											=					filter_non_english



		def 	__lowerCamelCase			(     self     :str  ):
						super().setUp()

						snake_case__	:      str     		=	[
						    '''[UNK]''',
						    '''[CLS]''',
						    '''[SEP]''',
						    '''[PAD]''',
						    '''[MASK]''',
						    '''want''',
						    '''##want''',
						    '''##ed''',
						    '''wa''',
						    '''un''',
						    '''runn''',
						    '''##ing''',
						    ''',''',
						    '''low''',
						    '''lowest''',
						]
						snake_case__	:      str     		=	os.path.join(self.tmpdirname   ,VOCAB_FILES_NAMES['''vocab_file''']  )
						with open(self.vocab_file   ,'''w'''   ,encoding='''utf-8'''  ) as vocab_writer:
										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )



		def 	__lowerCamelCase			(     self     :List[str]   ,__lowercase     :str  ):
						snake_case__	:      List[Any]     		=	'''UNwant\u00E9d,running'''
						snake_case__	:      str     		=	'''unwanted, running'''
						return input_text, output_text



		def 	__lowerCamelCase			(     self     :Optional[int]  ):
						snake_case__	:      str     		=	self.tokenizer_class(self.vocab_file  )

						snake_case__	:      str     		=	tokenizer.tokenize('''UNwant\u00E9d,running'''  )
						self.assertListEqual(__lowercase   ,['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase  )   ,[9, 6, 7, 1_2, 1_0, 1_1]  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						if not self.test_rust_tokenizer:
										return

						snake_case__	:      str     		=	self.get_tokenizer()
						snake_case__	:      Dict     		=	self.get_rust_tokenizer()

						snake_case__	:      List[Any]     		=	'''UNwant\u00E9d,running'''

						snake_case__	:      Dict     		=	tokenizer.tokenize(__lowercase  )
						snake_case__	:      int     		=	rust_tokenizer.tokenize(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      List[str]     		=	tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						snake_case__	:      List[str]     		=	rust_tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      Optional[Any]     		=	self.get_rust_tokenizer()
						snake_case__	:      List[Any]     		=	tokenizer.encode(__lowercase  )
						snake_case__	:      Dict     		=	rust_tokenizer.encode(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						# With lower casing
						snake_case__	:      List[Any]     		=	self.get_tokenizer(do_lower_case=__lowercase  )
						snake_case__	:      Optional[int]     		=	self.get_rust_tokenizer(do_lower_case=__lowercase  )

						snake_case__	:      int     		=	'''UNwant\u00E9d,running'''

						snake_case__	:      List[str]     		=	tokenizer.tokenize(__lowercase  )
						snake_case__	:      Union[str, Any]     		=	rust_tokenizer.tokenize(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      int     		=	tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						snake_case__	:      List[str]     		=	rust_tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      Tuple     		=	self.get_rust_tokenizer()
						snake_case__	:      int     		=	tokenizer.encode(__lowercase  )
						snake_case__	:      Dict     		=	rust_tokenizer.encode(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      Optional[Any]     		=	BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )   ,['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )



		def 	__lowerCamelCase			(     self     :Any  ):
						snake_case__	:      Dict     		=	BasicTokenizer(do_lower_case=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )   ,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''hello''']  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      Optional[int]     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''h\u00E9llo''']  )



		def 	__lowerCamelCase			(     self     :str  ):
						snake_case__	:      Union[str, Any]     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''hello''']  )



		def 	__lowerCamelCase			(     self     :Optional[Any]  ):
						snake_case__	:      Union[str, Any]     		=	BasicTokenizer(do_lower_case=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''hello''']  )



		def 	__lowerCamelCase			(     self     :Optional[Any]  ):
						snake_case__	:      str     		=	BasicTokenizer(do_lower_case=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )   ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      Any     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )



		def 	__lowerCamelCase			(     self     :int  ):
						snake_case__	:      str     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      List[Any]     		=	BasicTokenizer(do_lower_case=__lowercase   ,never_split=['''[UNK]''']  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )   ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )



		def 	__lowerCamelCase			(     self     :Dict  ):
						snake_case__	:      List[str]     		=	BasicTokenizer()
						snake_case__	:      Any     		=	'''a\n\'ll !!to?\'d of, can\'t.'''
						snake_case__	:      Optional[int]     		=	['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
						self.assertListEqual(tokenizer.tokenize(__lowercase  )   ,__lowercase  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      Any     		=	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

						snake_case__	:      Optional[Any]     		=	{}
						for i, token in enumerate(__lowercase  ):
										snake_case__	:      Dict     		=	i
						snake_case__	:      int     		=	WordpieceTokenizer(vocab=__lowercase   ,unk_token='''[UNK]'''  )

						self.assertListEqual(tokenizer.tokenize(''''''  )   ,[]  )

						self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )   ,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

						self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )   ,['''[UNK]''', '''runn''', '''##ing''']  )



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						self.assertTrue(_is_whitespace(''' '''  )  )
						self.assertTrue(_is_whitespace('''\t'''  )  )
						self.assertTrue(_is_whitespace('''\r'''  )  )
						self.assertTrue(_is_whitespace('''\n'''  )  )
						self.assertTrue(_is_whitespace('''\u00A0'''  )  )

						self.assertFalse(_is_whitespace('''A'''  )  )
						self.assertFalse(_is_whitespace('''-'''  )  )



		def 	__lowerCamelCase			(     self     :Union[str, Any]  ):
						self.assertTrue(_is_control('''\u0005'''  )  )

						self.assertFalse(_is_control('''A'''  )  )
						self.assertFalse(_is_control(''' '''  )  )
						self.assertFalse(_is_control('''\t'''  )  )
						self.assertFalse(_is_control('''\r'''  )  )



		def 	__lowerCamelCase			(     self     :str  ):
						self.assertTrue(_is_punctuation('''-'''  )  )
						self.assertTrue(_is_punctuation('''$'''  )  )
						self.assertTrue(_is_punctuation('''`'''  )  )
						self.assertTrue(_is_punctuation('''.'''  )  )

						self.assertFalse(_is_punctuation('''A'''  )  )
						self.assertFalse(_is_punctuation(''' '''  )  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      str     		=	self.get_tokenizer()
						snake_case__	:      List[str]     		=	self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(__lowercase  ) for t in ['''Test''', '''\xad''', '''test''']]   ,[['''[UNK]'''], [], ['''[UNK]''']]  )

						self.assertListEqual(
						    [rust_tokenizer.tokenize(__lowercase  ) for t in ['''Test''', '''\xad''', '''test''']]   ,[['''[UNK]'''], [], ['''[UNK]''']]  )



		@slow
		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      List[str]     		=	self.tokenizer_class.from_pretrained('''bert-base-uncased'''  )

						snake_case__	:      Union[str, Any]     		=	tokenizer.encode('''sequence builders'''   ,add_special_tokens=__lowercase  )
						snake_case__	:      str     		=	tokenizer.encode('''multi-sequence build'''   ,add_special_tokens=__lowercase  )

						snake_case__	:      Tuple     		=	tokenizer.build_inputs_with_special_tokens(__lowercase  )
						snake_case__	:      int     		=	tokenizer.build_inputs_with_special_tokens(__lowercase   ,__lowercase  )

						assert encoded_sentence == [1_0_1] + text + [1_0_2]
						assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
														snake_case__	:      str     		=	self.rust_tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )

														snake_case__	:      List[str]     		=	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
														snake_case__	:      List[Any]     		=	tokenizer_r.encode_plus(
														    __lowercase   ,return_attention_mask=__lowercase   ,return_token_type_ids=__lowercase   ,return_offsets_mapping=__lowercase   ,add_special_tokens=__lowercase   ,)

														snake_case__	:      List[str]     		=	tokenizer_r.do_lower_case if hasattr(__lowercase   ,'''do_lower_case'''  ) else False
														snake_case__	:      Optional[int]     		=	(
														    [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''A'''),
														        ((1, 2), ''','''),
														        ((3, 5), '''na'''),
														        ((5, 6), '''##ï'''),
														        ((6, 8), '''##ve'''),
														        ((9, 1_5), tokenizer_r.mask_token),
														        ((1_6, 2_1), '''Allen'''),
														        ((2_1, 2_3), '''##NL'''),
														        ((2_3, 2_4), '''##P'''),
														        ((2_5, 3_3), '''sentence'''),
														        ((3_3, 3_4), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														    if not do_lower_case
														    else [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''a'''),
														        ((1, 2), ''','''),
														        ((3, 8), '''naive'''),
														        ((9, 1_5), tokenizer_r.mask_token),
														        ((1_6, 2_1), '''allen'''),
														        ((2_1, 2_3), '''##nl'''),
														        ((2_3, 2_4), '''##p'''),
														        ((2_5, 3_3), '''sentence'''),
														        ((3_3, 3_4), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														)

														self.assertEqual(
														    [e[1] for e in expected_results]   ,tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
														self.assertEqual([e[0] for e in expected_results]   ,tokens['''offset_mapping''']  )






		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      str     		=	['''的''', '''人''', '''有''']
						snake_case__	:      Optional[int]     		=	''''''.join(__lowercase  )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
														snake_case__	:      Optional[Any]     		=	True
														snake_case__	:      int     		=	self.tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )
														snake_case__	:      Tuple     		=	self.rust_tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )

														snake_case__	:      Union[str, Any]     		=	tokenizer_p.encode(__lowercase   ,add_special_tokens=__lowercase  )
														snake_case__	:      List[str]     		=	tokenizer_r.encode(__lowercase   ,add_special_tokens=__lowercase  )

														snake_case__	:      List[str]     		=	tokenizer_r.convert_ids_to_tokens(__lowercase  )
														snake_case__	:      Optional[Any]     		=	tokenizer_p.convert_ids_to_tokens(__lowercase  )

														# it is expected that each Chinese character is not preceded by "##"
														self.assertListEqual(__lowercase   ,__lowercase  )
														self.assertListEqual(__lowercase   ,__lowercase  )

														snake_case__	:      str     		=	False
														snake_case__	:      Dict     		=	self.rust_tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )
														snake_case__	:      int     		=	self.tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )

														snake_case__	:      int     		=	tokenizer_r.encode(__lowercase   ,add_special_tokens=__lowercase  )
														snake_case__	:      Tuple     		=	tokenizer_p.encode(__lowercase   ,add_special_tokens=__lowercase  )

														snake_case__	:      List[Any]     		=	tokenizer_r.convert_ids_to_tokens(__lowercase  )
														snake_case__	:      Union[str, Any]     		=	tokenizer_p.convert_ids_to_tokens(__lowercase  )

														# it is expected that only the first Chinese character is not preceded by "##".
														snake_case__	:      Optional[int]     		=	[
														    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__lowercase  )
														]
														self.assertListEqual(__lowercase   ,__lowercase  )
														self.assertListEqual(__lowercase   ,__lowercase  )

