



import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision import transforms

from transformers import BitImageProcessor, FocalNetConfig, FocalNetForImageClassification
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling



def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					Any:


				"""simple docstring"""

				snake_case__	:      Tuple     		=	[2, 2, 6, 2] if '''tiny''' in model_name else [2, 2, 18, 2]
				snake_case__	:      int     		=	True if '''large''' in model_name or '''huge''' in model_name else False
				snake_case__	:      Union[str, Any]     		=	True if '''large''' in model_name or '''huge''' in model_name else False
				snake_case__	:      List[str]     		=	True if '''large''' in model_name or '''huge''' in model_name else False

				if "large" in model_name or "xlarge" in model_name or "huge" in model_name:
								if "fl3" in model_name:
												snake_case__	:      List[str]     		=	[3, 3, 3, 3]
												snake_case__	:      Optional[Any]     		=	[5, 5, 5, 5]
								elif "fl4" in model_name:
												snake_case__	:      List[str]     		=	[4, 4, 4, 4]
												snake_case__	:      int     		=	[3, 3, 3, 3]

				if "tiny" in model_name or "small" in model_name or "base" in model_name:
								snake_case__	:      Union[str, Any]     		=	[3, 3, 3, 3]
								if "lrf" in model_name:
												snake_case__	:      str     		=	[3, 3, 3, 3]
								else:
												snake_case__	:      Any     		=	[2, 2, 2, 2]

				if "tiny" in model_name:
								snake_case__	:      Optional[int]     		=	96
				elif "small" in model_name:
								snake_case__	:      str     		=	96
				elif "base" in model_name:
								snake_case__	:      Optional[int]     		=	128
				elif "large" in model_name:
								snake_case__	:      Any     		=	192
				elif "xlarge" in model_name:
								snake_case__	:      Optional[Any]     		=	256
				elif "huge" in model_name:
								snake_case__	:      Union[str, Any]     		=	352

				# set label information
				snake_case__	:      Tuple     		=	'''huggingface/label-files'''
				if "large" in model_name or "huge" in model_name:
								snake_case__	:      List[str]     		=	'''imagenet-22k-id2label.json'''
				else:
								snake_case__	:      Optional[Any]     		=	'''imagenet-1k-id2label.json'''

				snake_case__	:      Optional[Any]     		=	json.load(open(hf_hub_download(__lowerCAmelCase  ,  __lowerCAmelCase  ,  repo_type='''dataset'''      )  ,  '''r'''      )      )
				snake_case__	:      int     		=	{int(__lowerCAmelCase      ): v for k, v in idalabel.items()}
				snake_case__	:      List[Any]     		=	{v: k for k, v in idalabel.items()}

				snake_case__	:      List[Any]     		=	FocalNetConfig(
				    embed_dim=__lowerCAmelCase  ,  depths=__lowerCAmelCase  ,  focal_levels=__lowerCAmelCase  ,  focal_windows=__lowerCAmelCase  ,  use_conv_embed=__lowerCAmelCase  ,  idalabel=__lowerCAmelCase  ,  labelaid=__lowerCAmelCase  ,  use_post_layernorm=__lowerCAmelCase  ,  use_layerscale=__lowerCAmelCase  ,  )

				return config



def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					Tuple:


				"""simple docstring"""

				if "patch_embed.proj" in name:
								snake_case__	:      Optional[Any]     		=	name.replace('''patch_embed.proj'''  ,  '''embeddings.patch_embeddings.projection'''      )
				if "patch_embed.norm" in name:
								snake_case__	:      List[Any]     		=	name.replace('''patch_embed.norm'''  ,  '''embeddings.norm'''      )
				if "layers" in name:
								snake_case__	:      Union[str, Any]     		=	'''encoder.''' + name
				if "encoder.layers" in name:
								snake_case__	:      Union[str, Any]     		=	name.replace('''encoder.layers'''  ,  '''encoder.stages'''      )
				if "downsample.proj" in name:
								snake_case__	:      Union[str, Any]     		=	name.replace('''downsample.proj'''  ,  '''downsample.projection'''      )
				if "blocks" in name:
								snake_case__	:      Optional[int]     		=	name.replace('''blocks'''  ,  '''layers'''      )
				if "modulation.f.weight" in name or "modulation.f.bias" in name:
								snake_case__	:      Any     		=	name.replace('''modulation.f'''  ,  '''modulation.projection_in'''      )
				if "modulation.h.weight" in name or "modulation.h.bias" in name:
								snake_case__	:      Optional[int]     		=	name.replace('''modulation.h'''  ,  '''modulation.projection_context'''      )
				if "modulation.proj.weight" in name or "modulation.proj.bias" in name:
								snake_case__	:      Optional[int]     		=	name.replace('''modulation.proj'''  ,  '''modulation.projection_out'''      )

				if name == "norm.weight":
								snake_case__	:      Tuple     		=	'''layernorm.weight'''
				if name == "norm.bias":
								snake_case__	:      Tuple     		=	'''layernorm.bias'''

				if "head" in name:
								snake_case__	:      Any     		=	name.replace('''head'''  ,  '''classifier'''      )
				else:
								snake_case__	:      Optional[Any]     		=	'''focalnet.''' + name

				return name



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase=False      )   ->					Dict:


				"""simple docstring"""

				snake_case__	:      Optional[Any]     		=	{
				    '''focalnet-tiny''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_srf.pth''',
				    '''focalnet-tiny-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_lrf.pth''',
				    '''focalnet-small''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_srf.pth''',
				    '''focalnet-small-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_lrf.pth''',
				    '''focalnet-base''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_srf.pth''',
				    '''focalnet-base-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_lrf.pth''',
				    '''focalnet-large-lrf-fl3''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384.pth''',
				    '''focalnet-large-lrf-fl4''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384_fl4.pth''',
				    '''focalnet-xlarge-lrf-fl3''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384.pth''',
				    '''focalnet-xlarge-lrf-fl4''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384_fl4.pth''',
				}
				# fmt: on

				snake_case__	:      List[str]     		=	model_name_to_url[model_name]
				print('''Checkpoint URL: '''  ,  __lowerCAmelCase      )
				snake_case__	:      Union[str, Any]     		=	torch.hub.load_state_dict_from_url(__lowerCAmelCase  ,  map_location='''cpu'''      )['''model''']

				# rename keys
				for key in state_dict.copy().keys():
								snake_case__	:      List[str]     		=	state_dict.pop(__lowerCAmelCase      )
								snake_case__	:      List[str]     		=	val

				snake_case__	:      List[Any]     		=	get_focalnet_config(__lowerCAmelCase      )
				snake_case__	:      Dict     		=	FocalNetForImageClassification(__lowerCAmelCase      )
				model.eval()

				# load state dict
				model.load_state_dict(__lowerCAmelCase      )

				# verify conversion
				snake_case__	:      Optional[Any]     		=	'''http://images.cocodataset.org/val2017/000000039769.jpg'''

				snake_case__	:      Dict     		=	BitImageProcessor(
				    do_resize=__lowerCAmelCase  ,  size={'''shortest_edge''': 256}  ,  resample=PILImageResampling.BILINEAR  ,  do_center_crop=__lowerCAmelCase  ,  crop_size=224  ,  do_normalize=__lowerCAmelCase  ,  image_mean=__lowerCAmelCase  ,  image_std=__lowerCAmelCase  ,  )
				snake_case__	:      int     		=	Image.open(requests.get(__lowerCAmelCase  ,  stream=__lowerCAmelCase      ).raw      )
				snake_case__	:      Union[str, Any]     		=	processor(images=__lowerCAmelCase  ,  return_tensors='''pt'''      )

				snake_case__	:      str     		=	transforms.Compose(
				    [
				        transforms.Resize(256      ),
				        transforms.CenterCrop(224      ),
				        transforms.ToTensor(),
				        transforms.Normalize(mean=[0.485, 0.456, 0.406]  ,  std=[0.229, 0.224, 0.225]      ),
				    ]      )

				snake_case__	:      Union[str, Any]     		=	image_transforms(__lowerCAmelCase      ).unsqueeze(0      )

				# verify pixel_values
				assert torch.allclose(inputs.pixel_values  ,  __lowerCAmelCase  ,  atol=1E-4      )

				snake_case__	:      int     		=	model(**__lowerCAmelCase      )

				snake_case__	:      str     		=	outputs.logits.argmax(-1      ).item()
				print('''Predicted class:'''  ,  model.config.idalabel[predicted_class_idx]      )

				print('''First values of logits:'''  ,  outputs.logits[0, :3]      )

				if model_name == "focalnet-tiny":
								snake_case__	:      List[str]     		=	torch.tensor([0.2_166, -0.4_368, 0.2_191]      )
				elif model_name == "focalnet-tiny-lrf":
								snake_case__	:      Any     		=	torch.tensor([1.1_669, 0.0_125, -0.1_695]      )
				elif model_name == "focalnet-small":
								snake_case__	:      List[str]     		=	torch.tensor([0.4_917, -0.0_430, 0.1_341]      )
				elif model_name == "focalnet-small-lrf":
								snake_case__	:      Dict     		=	torch.tensor([-0.2_588, -0.5_342, -0.2_331]      )
				elif model_name == "focalnet-base":
								snake_case__	:      Tuple     		=	torch.tensor([-0.1_655, -0.4_090, -0.1_730]      )
				elif model_name == "focalnet-base-lrf":
								snake_case__	:      Dict     		=	torch.tensor([0.5_306, -0.0_483, -0.3_928]      )
				assert torch.allclose(outputs.logits[0, :3]  ,  __lowerCAmelCase  ,  atol=1E-4      )
				print('''Looks ok!'''      )

				if pytorch_dump_folder_path is not None:
								print(f"""Saving model and processor of {model_name} to {pytorch_dump_folder_path}"""      )
								model.save_pretrained(__lowerCAmelCase      )
								processor.save_pretrained(__lowerCAmelCase      )

				if push_to_hub:
								print(f"""Pushing model and processor of {model_name} to the hub..."""      )
								model.push_to_hub(f"""{model_name}"""      )
								processor.push_to_hub(f"""{model_name}"""      )


if __name__ == "__main__":
				A__								=						argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '''--model_name''',
				    default='''focalnet-tiny''',
				    type=str,
				    help='''Name of the FocalNet model you\'d like to convert.''',
				)
				parser.add_argument(
				    '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
				)
				parser.add_argument(
				    '''--push_to_hub''',
				    action='''store_true''',
				    help='''Whether to push the model and processor to the hub.''',
				)

				A__								=						parser.parse_args()
				convert_focalnet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class       a							(			__lowerCamelCase  ,							unittest.TestCase				):
		__lowerCAmelCase	:    int											=					BertTokenizer
		__lowerCAmelCase	:    int											=					BertTokenizerFast
		__lowerCAmelCase	:    int											=					True
		__lowerCAmelCase	:    List[Any]											=					True
		__lowerCAmelCase	:    Any											=					filter_non_english



		def 	__lowerCamelCase			(     self     :str  ):
						super().setUp()

						snake_case__	:      str     		=	[
						    '''[UNK]''',
						    '''[CLS]''',
						    '''[SEP]''',
						    '''[PAD]''',
						    '''[MASK]''',
						    '''want''',
						    '''##want''',
						    '''##ed''',
						    '''wa''',
						    '''un''',
						    '''runn''',
						    '''##ing''',
						    ''',''',
						    '''low''',
						    '''lowest''',
						]
						snake_case__	:      str     		=	os.path.join(self.tmpdirname   ,VOCAB_FILES_NAMES['''vocab_file''']  )
						with open(self.vocab_file   ,'''w'''   ,encoding='''utf-8'''  ) as vocab_writer:
										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )



		def 	__lowerCamelCase			(     self     :List[str]   ,__lowercase     :str  ):
						snake_case__	:      List[Any]     		=	'''UNwant\u00E9d,running'''
						snake_case__	:      str     		=	'''unwanted, running'''
						return input_text, output_text



		def 	__lowerCamelCase			(     self     :Optional[int]  ):
						snake_case__	:      str     		=	self.tokenizer_class(self.vocab_file  )

						snake_case__	:      str     		=	tokenizer.tokenize('''UNwant\u00E9d,running'''  )
						self.assertListEqual(__lowercase   ,['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase  )   ,[9, 6, 7, 1_2, 1_0, 1_1]  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						if not self.test_rust_tokenizer:
										return

						snake_case__	:      str     		=	self.get_tokenizer()
						snake_case__	:      Dict     		=	self.get_rust_tokenizer()

						snake_case__	:      List[Any]     		=	'''UNwant\u00E9d,running'''

						snake_case__	:      Dict     		=	tokenizer.tokenize(__lowercase  )
						snake_case__	:      int     		=	rust_tokenizer.tokenize(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      List[str]     		=	tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						snake_case__	:      List[str]     		=	rust_tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      Optional[Any]     		=	self.get_rust_tokenizer()
						snake_case__	:      List[Any]     		=	tokenizer.encode(__lowercase  )
						snake_case__	:      Dict     		=	rust_tokenizer.encode(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						# With lower casing
						snake_case__	:      List[Any]     		=	self.get_tokenizer(do_lower_case=__lowercase  )
						snake_case__	:      Optional[int]     		=	self.get_rust_tokenizer(do_lower_case=__lowercase  )

						snake_case__	:      int     		=	'''UNwant\u00E9d,running'''

						snake_case__	:      List[str]     		=	tokenizer.tokenize(__lowercase  )
						snake_case__	:      Union[str, Any]     		=	rust_tokenizer.tokenize(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      int     		=	tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						snake_case__	:      List[str]     		=	rust_tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      Tuple     		=	self.get_rust_tokenizer()
						snake_case__	:      int     		=	tokenizer.encode(__lowercase  )
						snake_case__	:      Dict     		=	rust_tokenizer.encode(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      Optional[Any]     		=	BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )   ,['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )



		def 	__lowerCamelCase			(     self     :Any  ):
						snake_case__	:      Dict     		=	BasicTokenizer(do_lower_case=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )   ,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''hello''']  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      Optional[int]     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''h\u00E9llo''']  )



		def 	__lowerCamelCase			(     self     :str  ):
						snake_case__	:      Union[str, Any]     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''hello''']  )



		def 	__lowerCamelCase			(     self     :Optional[Any]  ):
						snake_case__	:      Union[str, Any]     		=	BasicTokenizer(do_lower_case=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''hello''']  )



		def 	__lowerCamelCase			(     self     :Optional[Any]  ):
						snake_case__	:      str     		=	BasicTokenizer(do_lower_case=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )   ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      Any     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )



		def 	__lowerCamelCase			(     self     :int  ):
						snake_case__	:      str     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      List[Any]     		=	BasicTokenizer(do_lower_case=__lowercase   ,never_split=['''[UNK]''']  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )   ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )



		def 	__lowerCamelCase			(     self     :Dict  ):
						snake_case__	:      List[str]     		=	BasicTokenizer()
						snake_case__	:      Any     		=	'''a\n\'ll !!to?\'d of, can\'t.'''
						snake_case__	:      Optional[int]     		=	['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
						self.assertListEqual(tokenizer.tokenize(__lowercase  )   ,__lowercase  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      Any     		=	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

						snake_case__	:      Optional[Any]     		=	{}
						for i, token in enumerate(__lowercase  ):
										snake_case__	:      Dict     		=	i
						snake_case__	:      int     		=	WordpieceTokenizer(vocab=__lowercase   ,unk_token='''[UNK]'''  )

						self.assertListEqual(tokenizer.tokenize(''''''  )   ,[]  )

						self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )   ,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

						self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )   ,['''[UNK]''', '''runn''', '''##ing''']  )



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						self.assertTrue(_is_whitespace(''' '''  )  )
						self.assertTrue(_is_whitespace('''\t'''  )  )
						self.assertTrue(_is_whitespace('''\r'''  )  )
						self.assertTrue(_is_whitespace('''\n'''  )  )
						self.assertTrue(_is_whitespace('''\u00A0'''  )  )

						self.assertFalse(_is_whitespace('''A'''  )  )
						self.assertFalse(_is_whitespace('''-'''  )  )



		def 	__lowerCamelCase			(     self     :Union[str, Any]  ):
						self.assertTrue(_is_control('''\u0005'''  )  )

						self.assertFalse(_is_control('''A'''  )  )
						self.assertFalse(_is_control(''' '''  )  )
						self.assertFalse(_is_control('''\t'''  )  )
						self.assertFalse(_is_control('''\r'''  )  )



		def 	__lowerCamelCase			(     self     :str  ):
						self.assertTrue(_is_punctuation('''-'''  )  )
						self.assertTrue(_is_punctuation('''$'''  )  )
						self.assertTrue(_is_punctuation('''`'''  )  )
						self.assertTrue(_is_punctuation('''.'''  )  )

						self.assertFalse(_is_punctuation('''A'''  )  )
						self.assertFalse(_is_punctuation(''' '''  )  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      str     		=	self.get_tokenizer()
						snake_case__	:      List[str]     		=	self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(__lowercase  ) for t in ['''Test''', '''\xad''', '''test''']]   ,[['''[UNK]'''], [], ['''[UNK]''']]  )

						self.assertListEqual(
						    [rust_tokenizer.tokenize(__lowercase  ) for t in ['''Test''', '''\xad''', '''test''']]   ,[['''[UNK]'''], [], ['''[UNK]''']]  )



		@slow
		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      List[str]     		=	self.tokenizer_class.from_pretrained('''bert-base-uncased'''  )

						snake_case__	:      Union[str, Any]     		=	tokenizer.encode('''sequence builders'''   ,add_special_tokens=__lowercase  )
						snake_case__	:      str     		=	tokenizer.encode('''multi-sequence build'''   ,add_special_tokens=__lowercase  )

						snake_case__	:      Tuple     		=	tokenizer.build_inputs_with_special_tokens(__lowercase  )
						snake_case__	:      int     		=	tokenizer.build_inputs_with_special_tokens(__lowercase   ,__lowercase  )

						assert encoded_sentence == [1_0_1] + text + [1_0_2]
						assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
														snake_case__	:      str     		=	self.rust_tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )

														snake_case__	:      List[str]     		=	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
														snake_case__	:      List[Any]     		=	tokenizer_r.encode_plus(
														    __lowercase   ,return_attention_mask=__lowercase   ,return_token_type_ids=__lowercase   ,return_offsets_mapping=__lowercase   ,add_special_tokens=__lowercase   ,)

														snake_case__	:      List[str]     		=	tokenizer_r.do_lower_case if hasattr(__lowercase   ,'''do_lower_case'''  ) else False
														snake_case__	:      Optional[int]     		=	(
														    [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''A'''),
														        ((1, 2), ''','''),
														        ((3, 5), '''na'''),
														        ((5, 6), '''##ï'''),
														        ((6, 8), '''##ve'''),
														        ((9, 1_5), tokenizer_r.mask_token),
														        ((1_6, 2_1), '''Allen'''),
														        ((2_1, 2_3), '''##NL'''),
														        ((2_3, 2_4), '''##P'''),
														        ((2_5, 3_3), '''sentence'''),
														        ((3_3, 3_4), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														    if not do_lower_case
														    else [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''a'''),
														        ((1, 2), ''','''),
														        ((3, 8), '''naive'''),
														        ((9, 1_5), tokenizer_r.mask_token),
														        ((1_6, 2_1), '''allen'''),
														        ((2_1, 2_3), '''##nl'''),
														        ((2_3, 2_4), '''##p'''),
														        ((2_5, 3_3), '''sentence'''),
														        ((3_3, 3_4), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														)

														self.assertEqual(
														    [e[1] for e in expected_results]   ,tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
														self.assertEqual([e[0] for e in expected_results]   ,tokens['''offset_mapping''']  )






		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      str     		=	['''的''', '''人''', '''有''']
						snake_case__	:      Optional[int]     		=	''''''.join(__lowercase  )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
														snake_case__	:      Optional[Any]     		=	True
														snake_case__	:      int     		=	self.tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )
														snake_case__	:      Tuple     		=	self.rust_tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )

														snake_case__	:      Union[str, Any]     		=	tokenizer_p.encode(__lowercase   ,add_special_tokens=__lowercase  )
														snake_case__	:      List[str]     		=	tokenizer_r.encode(__lowercase   ,add_special_tokens=__lowercase  )

														snake_case__	:      List[str]     		=	tokenizer_r.convert_ids_to_tokens(__lowercase  )
														snake_case__	:      Optional[Any]     		=	tokenizer_p.convert_ids_to_tokens(__lowercase  )

														# it is expected that each Chinese character is not preceded by "##"
														self.assertListEqual(__lowercase   ,__lowercase  )
														self.assertListEqual(__lowercase   ,__lowercase  )

														snake_case__	:      str     		=	False
														snake_case__	:      Dict     		=	self.rust_tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )
														snake_case__	:      int     		=	self.tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )

														snake_case__	:      int     		=	tokenizer_r.encode(__lowercase   ,add_special_tokens=__lowercase  )
														snake_case__	:      Tuple     		=	tokenizer_p.encode(__lowercase   ,add_special_tokens=__lowercase  )

														snake_case__	:      List[Any]     		=	tokenizer_r.convert_ids_to_tokens(__lowercase  )
														snake_case__	:      Union[str, Any]     		=	tokenizer_p.convert_ids_to_tokens(__lowercase  )

														# it is expected that only the first Chinese character is not preceded by "##".
														snake_case__	:      Optional[int]     		=	[
														    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__lowercase  )
														]
														self.assertListEqual(__lowercase   ,__lowercase  )
														self.assertListEqual(__lowercase   ,__lowercase  )

