code: 
  
  
  
  
  
  
  """simple docstring"""
  
  from dataclasses import dataclass
  from typing import Any, Dict, Optional
  
  import torch
  import torch.nn.functional as F
  from torch import nn
  
  from ..configuration_utils import ConfigMixin, register_to_config
  from ..models.embeddings import ImagePositionalEmbeddings
  from ..utils import BaseOutput, deprecate
  from .attention import BasicTransformerBlock
  from .embeddings import PatchEmbed
  from .modeling_utils import ModelMixin
  
  
  @dataclass
  class 	__lowerCamelCase       (				A__    ):
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  		a_			:       torch.FloatTensor
  
  
  
  
  
  
  
  class 	__lowerCamelCase       (				A__				,							A__    ):
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  		@register_to_config
  		def __init__(     self							:			Union[str, Any]		,    a_							:			int = 16		,    a_							:			int = 88		,    a_							:			Optional[int] = None		,    a_							:			Optional[int] = None		,    a_							:			int = 1		,    a_							:			float = 0.0		,    a_							:			int = 32		,    a_							:			Optional[int] = None		,    a_							:			bool = False		,    a_							:			Optional[int] = None		,    a_							:			Optional[int] = None		,    a_							:			Optional[int] = None		,    a_							:			str = "geglu"		,    a_							:			Optional[int] = None		,    a_							:			bool = False		,    a_							:			bool = False		,    a_							:			bool = False		,    a_							:			str = "layer_norm"		,    a_							:			bool = True		,    ):
  				super().__init__()
  				lowerCAmelCase_     :							List[Any]             =    use_linear_projection
  				lowerCAmelCase_     :							Optional[int]             =    num_attention_heads
  				lowerCAmelCase_     :							str             =    attention_head_dim
  				lowerCAmelCase_     :							List[Any]             =    num_attention_heads * attention_head_dim
  
  				# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
  				# Define whether input is continuous or discrete depending on configuration
  				lowerCAmelCase_     :							int             =    (in_channels is not None) and (patch_size is None)
  				lowerCAmelCase_     :							Dict             =    num_vector_embeds is not None
  				lowerCAmelCase_     :							Optional[Any]             =    in_channels is not None and patch_size is not None
  
  				if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
  						lowerCAmelCase_     :							Dict             =    (
  						    f'''The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'''
  						    " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."
  						    " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"
  						    " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"
  						    " would be very nice if you could open a Pull request for the `transformer/config.json` file"
  						)
  						deprecate("norm_type!=num_embeds_ada_norm"		,    "1.0.0"		,    a_		,    standard_warn=a_			)
  						lowerCAmelCase_     :							str             =    "ada_norm"
  
  				if self.is_input_continuous and self.is_input_vectorized:
  						raise ValueError(
  						    f'''Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'''
  						    " sure that either `in_channels` or `num_vector_embeds` is None."			)
  				elif self.is_input_vectorized and self.is_input_patches:
  						raise ValueError(
  						    f'''Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'''
  						    " sure that either `num_vector_embeds` or `num_patches` is None."			)
  				elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
  						raise ValueError(
  						    f'''Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'''
  						    f''' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.'''			)
  
  				# 2. Define input layers
  				if self.is_input_continuous:
  						lowerCAmelCase_     :							Optional[int]             =    in_channels
  
  						lowerCAmelCase_     :							List[str]             =    torch.nn.GroupNorm(num_groups=a_		,    num_channels=a_		,    eps=1e-6		,    affine=a_			)
  						if use_linear_projection:
  								lowerCAmelCase_     :							Union[str, Any]             =    nn.Linear(a_		,    a_			)
  						else:
  								lowerCAmelCase_     :							Union[str, Any]             =    nn.Convad(a_		,    a_		,    kernel_size=1		,    stride=1		,    padding=0			)
  				elif self.is_input_vectorized:
  						assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
  						assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"
  
  						lowerCAmelCase_     :							Optional[Any]             =    sample_size
  						lowerCAmelCase_     :							int             =    sample_size
  						lowerCAmelCase_     :							Union[str, Any]             =    num_vector_embeds
  						lowerCAmelCase_     :							Union[str, Any]             =    self.height * self.width
  
  						lowerCAmelCase_     :							int             =    ImagePositionalEmbeddings(
  						    num_embed=a_		,    embed_dim=a_		,    height=self.height		,    width=self.width			)
  				elif self.is_input_patches:
  						assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"
  
  						lowerCAmelCase_     :							List[Any]             =    sample_size
  						lowerCAmelCase_     :							List[Any]             =    sample_size
  
  						lowerCAmelCase_     :							int             =    patch_size
  						lowerCAmelCase_     :							str             =    PatchEmbed(
  						    height=a_		,    width=a_		,    patch_size=a_		,    in_channels=a_		,    embed_dim=a_		,    )
  
  				# 3. Define transformers blocks
  				lowerCAmelCase_     :							Dict             =    nn.ModuleList(
  				    [
  				        BasicTransformerBlock(
  				            a_		,    a_		,    a_		,    dropout=a_		,    cross_attention_dim=a_		,    activation_fn=a_		,    num_embeds_ada_norm=a_		,    attention_bias=a_		,    only_cross_attention=a_		,    upcast_attention=a_		,    norm_type=a_		,    norm_elementwise_affine=a_		,    )
  				        for d in range(a_			)
  				    ]			)
  
  				# 4. Define output layers
  				lowerCAmelCase_     :							Optional[int]             =    in_channels if out_channels is None else out_channels
  				if self.is_input_continuous:
  						# TODO: should use out_channels for continuous projections
  						if use_linear_projection:
  								lowerCAmelCase_     :							Tuple             =    nn.Linear(a_		,    a_			)
  						else:
  								lowerCAmelCase_     :							int             =    nn.Convad(a_		,    a_		,    kernel_size=1		,    stride=1		,    padding=0			)
  				elif self.is_input_vectorized:
  						lowerCAmelCase_     :							Optional[int]             =    nn.LayerNorm(a_			)
  						lowerCAmelCase_     :							Optional[int]             =    nn.Linear(a_		,    self.num_vector_embeds - 1			)
  				elif self.is_input_patches:
  						lowerCAmelCase_     :							List[Any]             =    nn.LayerNorm(a_		,    elementwise_affine=a_		,    eps=1e-6			)
  						lowerCAmelCase_     :							List[str]             =    nn.Linear(a_		,    2 * inner_dim			)
  						lowerCAmelCase_     :							Dict             =    nn.Linear(a_		,    patch_size * patch_size * self.out_channels			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Union[str, Any]		,    a_							:			torch.Tensor		,    a_							:			Optional[torch.Tensor] = None		,    a_							:			Optional[torch.LongTensor] = None		,    a_							:			Optional[torch.LongTensor] = None		,    a_							:			Dict[str, Any] = None		,    a_							:			Optional[torch.Tensor] = None		,    a_							:			Optional[torch.Tensor] = None		,    a_							:			bool = True		,    ):
  				# ensure attention_mask is a bias, and give it a singleton query_tokens dimension.
  				#   we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.
  				#   we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.
  				# expects mask of shape:
  				#   [batch, key_tokens]
  				# adds singleton query_tokens dimension:
  				#   [batch,                    1, key_tokens]
  				# this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
  				#   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
  				#   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
  				if attention_mask is not None and attention_mask.ndim == 2:
  						# assume that mask is expressed as:
  						#   (1 = keep,      0 = discard)
  						# convert mask into a bias that can be added to attention scores:
  						#       (keep = +0,     discard = -10000.0)
  						lowerCAmelCase_     :							str             =    (1 - attention_mask.to(hidden_states.dtype			)) * -10000.0
  						lowerCAmelCase_     :							Any             =    attention_mask.unsqueeze(1			)
  
  				# convert encoder_attention_mask to a bias the same way we do for attention_mask
  				if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
  						lowerCAmelCase_     :							List[str]             =    (1 - encoder_attention_mask.to(hidden_states.dtype			)) * -10000.0
  						lowerCAmelCase_     :							Union[str, Any]             =    encoder_attention_mask.unsqueeze(1			)
  
  				# 1. Input
  				if self.is_input_continuous:
  						lowerCAmelCase_       ,			lowerCAmelCase_       ,			lowerCAmelCase_       ,			lowerCAmelCase_     :							str             =    hidden_states.shape
  						lowerCAmelCase_     :							Union[str, Any]             =    hidden_states
  
  						lowerCAmelCase_     :							int             =    self.norm(a_			)
  						if not self.use_linear_projection:
  								lowerCAmelCase_     :							Optional[Any]             =    self.proj_in(a_			)
  								lowerCAmelCase_     :							List[Any]             =    hidden_states.shape[1]
  								lowerCAmelCase_     :							Optional[Any]             =    hidden_states.permute(0		,    2		,    3		,    1			).reshape(a_		,    height * width		,    a_			)
  						else:
  								lowerCAmelCase_     :							int             =    hidden_states.shape[1]
  								lowerCAmelCase_     :							Dict             =    hidden_states.permute(0		,    2		,    3		,    1			).reshape(a_		,    height * width		,    a_			)
  								lowerCAmelCase_     :							Tuple             =    self.proj_in(a_			)
  				elif self.is_input_vectorized:
  						lowerCAmelCase_     :							List[str]             =    self.latent_image_embedding(a_			)
  				elif self.is_input_patches:
  						lowerCAmelCase_     :							int             =    self.pos_embed(a_			)
  
  				# 2. Blocks
  				for block in self.transformer_blocks:
  						lowerCAmelCase_     :							Optional[Any]             =    block(
  						    a_		,    attention_mask=a_		,    encoder_hidden_states=a_		,    encoder_attention_mask=a_		,    timestep=a_		,    cross_attention_kwargs=a_		,    class_labels=a_		,    )
  
  				# 3. Output
  				if self.is_input_continuous:
  						if not self.use_linear_projection:
  								lowerCAmelCase_     :							str             =    hidden_states.reshape(a_		,    a_		,    a_		,    a_			).permute(0		,    3		,    1		,    2			).contiguous()
  								lowerCAmelCase_     :							Optional[Any]             =    self.proj_out(a_			)
  						else:
  								lowerCAmelCase_     :							List[Any]             =    self.proj_out(a_			)
  								lowerCAmelCase_     :							str             =    hidden_states.reshape(a_		,    a_		,    a_		,    a_			).permute(0		,    3		,    1		,    2			).contiguous()
  
  						lowerCAmelCase_     :							List[str]             =    hidden_states + residual
  				elif self.is_input_vectorized:
  						lowerCAmelCase_     :							Tuple             =    self.norm_out(a_			)
  						lowerCAmelCase_     :							List[Any]             =    self.out(a_			)
  						# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
  						lowerCAmelCase_     :							Optional[int]             =    logits.permute(0		,    2		,    1			)
  
  						# log(p(x_0))
  						lowerCAmelCase_     :							List[Any]             =    F.log_softmax(logits.double()		,    dim=1			).float()
  				elif self.is_input_patches:
  						# TODO: cleanup!
  						lowerCAmelCase_     :							str             =    self.transformer_blocks[0].norma.emb(
  						    a_		,    a_		,    hidden_dtype=hidden_states.dtype			)
  						lowerCAmelCase_       ,			lowerCAmelCase_     :							int             =    self.proj_out_a(F.silu(a_			)			).chunk(2		,    dim=1			)
  						lowerCAmelCase_     :							List[Any]             =    self.norm_out(a_			) * (1 + scale[:, None]) + shift[:, None]
  						lowerCAmelCase_     :							Union[str, Any]             =    self.proj_out_a(a_			)
  
  						# unpatchify
  						lowerCAmelCase_     :							Tuple             =    int(hidden_states.shape[1] ** 0.5			)
  						lowerCAmelCase_     :							Dict             =    hidden_states.reshape(
  						    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)			)
  						lowerCAmelCase_     :							str             =    torch.einsum("nhwpqc->nchpwq"		,    a_			)
  						lowerCAmelCase_     :							Optional[int]             =    hidden_states.reshape(
  						    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)			)
  
  				if not return_dict:
  						return (output,)
  
  				return TransformeraDModelOutput(sample=a_			)
  
style_context: 
  
  
  
  
  
  
  """simple docstring"""
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  @require_tokenizers
  class 	__lowerCamelCase       (				A__				,							unittest.TestCase    ):
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  		a_			:       List[str]      =							RealmTokenizer
  		a_			:       int      =							RealmTokenizerFast
  		a_			:       str      =							True
  		a_			:       Tuple      =							True
  		a_			:       List[Any]      =							filter_non_english
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				super().setUp()
  
  				lowerCAmelCase_     :							int             =    [
  				    "[UNK]",
  				    "[CLS]",
  				    "[SEP]",
  				    "[PAD]",
  				    "[MASK]",
  				    "want",
  				    "##want",
  				    "##ed",
  				    "wa",
  				    "un",
  				    "runn",
  				    "##ing",
  				    ",",
  				    "low",
  				    "lowest",
  				]
  				lowerCAmelCase_     :							List[Any]             =    os.path.join(self.tmpdirname		,    VOCAB_FILES_NAMES["vocab_file"]			)
  				with open(self.vocab_file		,    "w"		,    encoding="utf-8"			) as vocab_writer:
  						vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]		,    a_							:			Any			):
  				lowerCAmelCase_     :							int             =    "UNwant\u00E9d,running"
  				lowerCAmelCase_     :							Optional[Any]             =    "unwanted, running"
  				return input_text, output_text
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_     :							Tuple             =    self.tokenizer_class(self.vocab_file			)
  
  				lowerCAmelCase_     :							List[str]             =    tokenizer.tokenize("UNwant\u00E9d,running"			)
  				self.assertListEqual(a_		,    ["un", "##want", "##ed", ",", "runn", "##ing"]			)
  				self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)		,    [9, 6, 7, 12, 10, 11]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Dict			):
  				if not self.test_rust_tokenizer:
  						return
  
  				lowerCAmelCase_     :							str             =    self.get_tokenizer()
  				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer()
  
  				lowerCAmelCase_     :							Optional[Any]             =    "UNwant\u00E9d,running"
  
  				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.tokenize(a_			)
  				lowerCAmelCase_     :							List[Any]             =    rust_tokenizer.tokenize(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							str             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							int             =    self.get_rust_tokenizer()
  				lowerCAmelCase_     :							Any             =    tokenizer.encode(a_			)
  				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.encode(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				# With lower casing
  				lowerCAmelCase_     :							Dict             =    self.get_tokenizer(do_lower_case=a_			)
  				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer(do_lower_case=a_			)
  
  				lowerCAmelCase_     :							Dict             =    "UNwant\u00E9d,running"
  
  				lowerCAmelCase_     :							Union[str, Any]             =    tokenizer.tokenize(a_			)
  				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.tokenize(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
  				lowerCAmelCase_     :							str             =    tokenizer.encode(a_			)
  				lowerCAmelCase_     :							Optional[int]             =    rust_tokenizer.encode(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]			):
  				lowerCAmelCase_     :							Any             =    BasicTokenizer()
  
  				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)		,    ["ah", "\u535A", "\u63A8", "zz"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["hello", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				lowerCAmelCase_     :							Optional[int]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hällo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["h\u00E9llo"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[Any]			):
  				lowerCAmelCase_     :							Optional[Any]             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[Any]			):
  				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HäLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Dict			):
  				lowerCAmelCase_     :							List[str]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HaLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[str]			):
  				lowerCAmelCase_     :							Any             =    BasicTokenizer(do_lower_case=a_		,    never_split=["[UNK]"]			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				lowerCAmelCase_     :							int             =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
  				lowerCAmelCase_     :							Union[str, Any]             =    {}
  				for i, token in enumerate(a_			):
  						lowerCAmelCase_     :							Any             =    i
  				lowerCAmelCase_     :							int             =    WordpieceTokenizer(vocab=a_		,    unk_token="[UNK]"			)
  
  				self.assertListEqual(tokenizer.tokenize(""			)		,    []			)
  
  				self.assertListEqual(tokenizer.tokenize("unwanted running"			)		,    ["un", "##want", "##ed", "runn", "##ing"]			)
  
  				self.assertListEqual(tokenizer.tokenize("unwantedX running"			)		,    ["[UNK]", "runn", "##ing"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				self.assertTrue(_is_whitespace(" "			)			)
  				self.assertTrue(_is_whitespace("\t"			)			)
  				self.assertTrue(_is_whitespace("\r"			)			)
  				self.assertTrue(_is_whitespace("\n"			)			)
  				self.assertTrue(_is_whitespace("\u00A0"			)			)
  
  				self.assertFalse(_is_whitespace("A"			)			)
  				self.assertFalse(_is_whitespace("-"			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				self.assertTrue(_is_control("\u0005"			)			)
  
  				self.assertFalse(_is_control("A"			)			)
  				self.assertFalse(_is_control(" "			)			)
  				self.assertFalse(_is_control("\t"			)			)
  				self.assertFalse(_is_control("\r"			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				self.assertTrue(_is_punctuation("-"			)			)
  				self.assertTrue(_is_punctuation("$"			)			)
  				self.assertTrue(_is_punctuation("`"			)			)
  				self.assertTrue(_is_punctuation("."			)			)
  
  				self.assertFalse(_is_punctuation("A"			)			)
  				self.assertFalse(_is_punctuation(" "			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				lowerCAmelCase_     :							Any             =    self.get_tokenizer()
  
  				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  				self.assertListEqual([tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)
  
  				if self.test_rust_tokenizer:
  						lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
  						self.assertListEqual(
  						    [rust_tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)
  
  
  
  
  
  		@slow
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_     :							int             =    self.tokenizer_class.from_pretrained("bert-base-uncased"			)
  
  				lowerCAmelCase_     :							List[Any]             =    tokenizer.encode("sequence builders"		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Optional[int]             =    tokenizer.encode("multi-sequence build"		,    add_special_tokens=a_			)
  
  				lowerCAmelCase_     :							Dict             =    tokenizer.build_inputs_with_special_tokens(a_			)
  				lowerCAmelCase_     :							Tuple             =    tokenizer.build_inputs_with_special_tokens(a_		,    a_			)
  
  				assert encoded_sentence == [1_01] + text + [1_02]
  				assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]			):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
  								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)
  
  								lowerCAmelCase_     :							Tuple             =    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  								lowerCAmelCase_     :							List[str]             =    tokenizer_r.encode_plus(
  								    a_		,    return_attention_mask=a_		,    return_token_type_ids=a_		,    return_offsets_mapping=a_		,    add_special_tokens=a_		,    )
  
  								lowerCAmelCase_     :							List[str]             =    tokenizer_r.do_lower_case if hasattr(a_		,    "do_lower_case"			) else False
  								lowerCAmelCase_     :							int             =    (
  								    [
  								        ((0, 0), tokenizer_r.cls_token),
  								        ((0, 1), "A"),
  								        ((1, 2), ","),
  								        ((3, 5), "na"),
  								        ((5, 6), "##ï"),
  								        ((6, 8), "##ve"),
  								        ((9, 15), tokenizer_r.mask_token),
  								        ((16, 21), "Allen"),
  								        ((21, 23), "##NL"),
  								        ((23, 24), "##P"),
  								        ((25, 33), "sentence"),
  								        ((33, 34), "."),
  								        ((0, 0), tokenizer_r.sep_token),
  								    ]
  								    if not do_lower_case
  								    else [
  								        ((0, 0), tokenizer_r.cls_token),
  								        ((0, 1), "a"),
  								        ((1, 2), ","),
  								        ((3, 8), "naive"),
  								        ((9, 15), tokenizer_r.mask_token),
  								        ((16, 21), "allen"),
  								        ((21, 23), "##nl"),
  								        ((23, 24), "##p"),
  								        ((25, 33), "sentence"),
  								        ((33, 34), "."),
  								        ((0, 0), tokenizer_r.sep_token),
  								    ]
  								)
  
  								self.assertEqual(
  								    [e[1] for e in expected_results]		,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
  								self.assertEqual([e[0] for e in expected_results]		,    tokens["offset_mapping"]			)
  
  
  
  
  
  		@slow
  		def      lowerCamelCase (     self							:			int			):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
  								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)
  								lowerCAmelCase_     :							Tuple             =    self.tokenizer_class.from_pretrained(a_		,    **a_			)
  								lowerCAmelCase_     :							Dict             =    [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]
  
  								lowerCAmelCase_     :							Optional[Any]             =    tokenizer_r.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)
  								lowerCAmelCase_     :							int             =    tokenizer_p.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)
  
  								lowerCAmelCase_     :							Dict             =    (2, 2, 10)
  
  								self.assertEqual(encoded_sentence_r["input_ids"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_r["attention_mask"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_r["token_type_ids"].shape		,    a_			)
  
  								self.assertEqual(encoded_sentence_p["input_ids"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_p["attention_mask"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_p["token_type_ids"].shape		,    a_			)
  