
import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import SegformerImageProcessor, SwinConfig, UperNetConfig, UperNetForSemanticSegmentation
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Optional[Any]:
			lowerCAmelCase__      :      Union[str, Any]         =		384
			lowerCAmelCase__      :      Tuple         =		7
			if "tiny" in model_name:
						lowerCAmelCase__      :      Union[str, Any]         =		96
						lowerCAmelCase__      :      Optional[int]         =		(2, 2, 6, 2)
						lowerCAmelCase__      :      Dict         =		(3, 6, 12, 24)
			elif "small" in model_name:
						lowerCAmelCase__      :      int         =		96
						lowerCAmelCase__      :      List[Any]         =		(2, 2, 18, 2)
						lowerCAmelCase__      :      Optional[Any]         =		(3, 6, 12, 24)
			elif "base" in model_name:
						lowerCAmelCase__      :      List[Any]         =		128
						lowerCAmelCase__      :      Optional[int]         =		(2, 2, 18, 2)
						lowerCAmelCase__      :      str         =		(4, 8, 16, 32)
						lowerCAmelCase__      :      int         =		12
						lowerCAmelCase__      :      int         =		512
			elif "large" in model_name:
						lowerCAmelCase__      :      Optional[int]         =		192
						lowerCAmelCase__      :      Dict         =		(2, 2, 18, 2)
						lowerCAmelCase__      :      Tuple         =		(6, 12, 24, 48)
						lowerCAmelCase__      :      Optional[Any]         =		12
						lowerCAmelCase__      :      int         =		768

			# set label information
			lowerCAmelCase__      :      Tuple         =		150
			lowerCAmelCase__      :      List[Any]         =		'huggingface/label-files'
			lowerCAmelCase__      :      Optional[Any]         =		'ade20k-id2label.json'
			lowerCAmelCase__      :      Any         =		json.load(open(hf_hub_download(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							repo_type='dataset'     )       ,							'r'     )     )
			lowerCAmelCase__      :      Union[str, Any]         =		{int(SCREAMING_SNAKE_CASE_     ): v for k, v in idalabel.items()}
			lowerCAmelCase__      :      List[str]         =		{v: k for k, v in idalabel.items()}

			lowerCAmelCase__      :      str         =		SwinConfig(
			    embed_dim=SCREAMING_SNAKE_CASE_       ,							depths=SCREAMING_SNAKE_CASE_       ,							num_heads=SCREAMING_SNAKE_CASE_       ,							window_size=SCREAMING_SNAKE_CASE_       ,							out_features=['stage1', 'stage2', 'stage3', 'stage4']       ,							)
			lowerCAmelCase__      :      Tuple         =		UperNetConfig(
			    backbone_config=SCREAMING_SNAKE_CASE_       ,							auxiliary_in_channels=SCREAMING_SNAKE_CASE_       ,							num_labels=SCREAMING_SNAKE_CASE_       ,							idalabel=SCREAMING_SNAKE_CASE_       ,							labelaid=SCREAMING_SNAKE_CASE_       ,							)

			return config
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		int:
			lowerCAmelCase__      :      int         =		[]

			# fmt: off
			# stem
			rename_keys.append(('backbone.patch_embed.projection.weight', 'backbone.embeddings.patch_embeddings.projection.weight')     )
			rename_keys.append(('backbone.patch_embed.projection.bias', 'backbone.embeddings.patch_embeddings.projection.bias')     )
			rename_keys.append(('backbone.patch_embed.norm.weight', 'backbone.embeddings.norm.weight')     )
			rename_keys.append(('backbone.patch_embed.norm.bias', 'backbone.embeddings.norm.bias')     )
			# stages
			for i in range(len(config.backbone_config.depths     )     ):
						for j in range(config.backbone_config.depths[i]     ):
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.norm1.weight''', F'''backbone.encoder.layers.{i}.blocks.{j}.layernorm_before.weight''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.norm1.bias''', F'''backbone.encoder.layers.{i}.blocks.{j}.layernorm_before.bias''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.attn.w_msa.relative_position_bias_table''', F'''backbone.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_bias_table''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.attn.w_msa.relative_position_index''', F'''backbone.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_index''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.attn.w_msa.proj.weight''', F'''backbone.encoder.layers.{i}.blocks.{j}.attention.output.dense.weight''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.attn.w_msa.proj.bias''', F'''backbone.encoder.layers.{i}.blocks.{j}.attention.output.dense.bias''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.norm2.weight''', F'''backbone.encoder.layers.{i}.blocks.{j}.layernorm_after.weight''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.norm2.bias''', F'''backbone.encoder.layers.{i}.blocks.{j}.layernorm_after.bias''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.ffn.layers.0.0.weight''', F'''backbone.encoder.layers.{i}.blocks.{j}.intermediate.dense.weight''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.ffn.layers.0.0.bias''', F'''backbone.encoder.layers.{i}.blocks.{j}.intermediate.dense.bias''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.ffn.layers.1.weight''', F'''backbone.encoder.layers.{i}.blocks.{j}.output.dense.weight''')     )
									rename_keys.append((F'''backbone.stages.{i}.blocks.{j}.ffn.layers.1.bias''', F'''backbone.encoder.layers.{i}.blocks.{j}.output.dense.bias''')     )

						if i < 3:
									rename_keys.append((F'''backbone.stages.{i}.downsample.reduction.weight''', F'''backbone.encoder.layers.{i}.downsample.reduction.weight''')     )
									rename_keys.append((F'''backbone.stages.{i}.downsample.norm.weight''', F'''backbone.encoder.layers.{i}.downsample.norm.weight''')     )
									rename_keys.append((F'''backbone.stages.{i}.downsample.norm.bias''', F'''backbone.encoder.layers.{i}.downsample.norm.bias''')     )
						rename_keys.append((F'''backbone.norm{i}.weight''', F'''backbone.hidden_states_norms.stage{i+1}.weight''')     )
						rename_keys.append((F'''backbone.norm{i}.bias''', F'''backbone.hidden_states_norms.stage{i+1}.bias''')     )

			# decode head
			rename_keys.extend(
			    [
			        ('decode_head.conv_seg.weight', 'decode_head.classifier.weight'),
			        ('decode_head.conv_seg.bias', 'decode_head.classifier.bias'),
			        ('auxiliary_head.conv_seg.weight', 'auxiliary_head.classifier.weight'),
			        ('auxiliary_head.conv_seg.bias', 'auxiliary_head.classifier.bias'),
			    ]     )
			# fmt: on

			return rename_keys
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Optional[int]:
			lowerCAmelCase__      :      List[Any]         =		dct.pop(SCREAMING_SNAKE_CASE_     )
			lowerCAmelCase__      :      Optional[int]         =		val
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Dict:
			lowerCAmelCase__      :      Dict         =		[int(backbone_config.embed_dim * 2**i     ) for i in range(len(backbone_config.depths     )     )]
			for i in range(len(backbone_config.depths     )     ):
						lowerCAmelCase__      :      Union[str, Any]         =		num_features[i]
						for j in range(backbone_config.depths[i]     ):
									# fmt: off
									# read in weights + bias of input projection layer (in original implementation, this is a single matrix + bias)
									lowerCAmelCase__      :      int         =		state_dict.pop(F'''backbone.stages.{i}.blocks.{j}.attn.w_msa.qkv.weight'''     )
									lowerCAmelCase__      :      List[str]         =		state_dict.pop(F'''backbone.stages.{i}.blocks.{j}.attn.w_msa.qkv.bias'''     )
									# next, add query, keys and values (in that order) to the state dict
									lowerCAmelCase__      :      Any         =		in_proj_weight[:dim, :]
									lowerCAmelCase__      :      List[Any]         =		in_proj_bias[: dim]
									lowerCAmelCase__      :      Dict         =		in_proj_weight[
									    dim : dim * 2, :
									]
									lowerCAmelCase__      :      Optional[int]         =		in_proj_bias[
									    dim : dim * 2
									]
									lowerCAmelCase__      :      str         =		in_proj_weight[
									    -dim :, :
									]
									lowerCAmelCase__      :      int         =		in_proj_bias[-dim :]
									# fmt: on
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Optional[Any]:
			lowerCAmelCase__      ,				lowerCAmelCase__      :      Any         =		x.shape
			lowerCAmelCase__      :      Union[str, Any]         =		x.reshape(SCREAMING_SNAKE_CASE_       ,							4       ,							in_channel // 4     )
			lowerCAmelCase__      :      Optional[Any]         =		x[:, [0, 2, 1, 3], :].transpose(1       ,							2     ).reshape(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )
			return x
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		List[Any]:
			lowerCAmelCase__      ,				lowerCAmelCase__      :      Optional[Any]         =		x.shape
			lowerCAmelCase__      :      Optional[Any]         =		x.reshape(SCREAMING_SNAKE_CASE_       ,							in_channel // 4       ,							4     )
			lowerCAmelCase__      :      Optional[Any]         =		x[:, :, [0, 2, 1, 3]].transpose(1       ,							2     ).reshape(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )

			return x
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Any:
			lowerCAmelCase__      :      Optional[Any]         =		x.shape[0]
			lowerCAmelCase__      :      Any         =		x.reshape(4       ,							in_channel // 4     )
			lowerCAmelCase__      :      Tuple         =		x[[0, 2, 1, 3], :].transpose(0       ,							1     ).reshape(SCREAMING_SNAKE_CASE_     )
			return x
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		List[Any]:
			lowerCAmelCase__      :      List[str]         =		x.shape[0]
			lowerCAmelCase__      :      int         =		x.reshape(in_channel // 4       ,							4     )
			lowerCAmelCase__      :      Tuple         =		x[:, [0, 2, 1, 3]].transpose(0       ,							1     ).reshape(SCREAMING_SNAKE_CASE_     )
			return x
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		str:
			lowerCAmelCase__      :      Dict         =		{
			    'upernet-swin-tiny': 'https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210531_112542-e380ad3e.pth',
			    'upernet-swin-small': 'https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_small_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K/upernet_swin_small_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210526_192015-ee2fff1c.pth',
			    'upernet-swin-base': 'https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_base_patch4_window12_512x512_160k_ade20k_pretrain_384x384_22K/upernet_swin_base_patch4_window12_512x512_160k_ade20k_pretrain_384x384_22K_20210531_125459-429057bf.pth',
			    'upernet-swin-large': 'https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_large_patch4_window12_512x512_pretrain_384x384_22K_160k_ade20k/upernet_swin_large_patch4_window12_512x512_pretrain_384x384_22K_160k_ade20k_20220318_091743-9ba68901.pth',
			}
			lowerCAmelCase__      :      Optional[Any]         =		model_name_to_url[model_name]
			lowerCAmelCase__      :      int         =		torch.hub.load_state_dict_from_url(SCREAMING_SNAKE_CASE_       ,							map_location='cpu'       ,							file_name=SCREAMING_SNAKE_CASE_     )[
			    'state_dict'
			]

			for name, param in state_dict.items():
						print(SCREAMING_SNAKE_CASE_       ,							param.shape     )

			lowerCAmelCase__      :      int         =		get_upernet_config(SCREAMING_SNAKE_CASE_     )
			lowerCAmelCase__      :      Tuple         =		UperNetForSemanticSegmentation(SCREAMING_SNAKE_CASE_     )
			model.eval()

			# replace "bn" => "batch_norm"
			for key in state_dict.copy().keys():
						lowerCAmelCase__      :      Optional[Any]         =		state_dict.pop(SCREAMING_SNAKE_CASE_     )
						if "bn" in key:
									lowerCAmelCase__      :      List[str]         =		key.replace('bn'       ,							'batch_norm'     )
						lowerCAmelCase__      :      Any         =		val

			# rename keys
			lowerCAmelCase__      :      Optional[Any]         =		create_rename_keys(SCREAMING_SNAKE_CASE_     )
			for src, dest in rename_keys:
						rename_key(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )
			read_in_q_k_v(SCREAMING_SNAKE_CASE_       ,							config.backbone_config     )

			# fix downsample parameters
			for key, value in state_dict.items():
						if "downsample" in key:
									if "reduction" in key:
												lowerCAmelCase__      :      int         =		reverse_correct_unfold_reduction_order(SCREAMING_SNAKE_CASE_     )
									if "norm" in key:
												lowerCAmelCase__      :      Dict         =		reverse_correct_unfold_norm_order(SCREAMING_SNAKE_CASE_     )

			model.load_state_dict(SCREAMING_SNAKE_CASE_     )

			# verify on image
			lowerCAmelCase__      :      Dict         =		'https://huggingface.co/datasets/hf-internal-testing/fixtures_ade20k/resolve/main/ADE_val_00000001.jpg'
			lowerCAmelCase__      :      Dict         =		Image.open(requests.get(SCREAMING_SNAKE_CASE_       ,							stream=SCREAMING_SNAKE_CASE_     ).raw     ).convert('RGB'     )

			lowerCAmelCase__      :      Tuple         =		SegformerImageProcessor()
			lowerCAmelCase__      :      Optional[int]         =		processor(SCREAMING_SNAKE_CASE_       ,							return_tensors='pt'     ).pixel_values

			with torch.no_grad():
						lowerCAmelCase__      :      Any         =		model(SCREAMING_SNAKE_CASE_     )
						lowerCAmelCase__      :      Optional[Any]         =		outputs.logits

			print(logits.shape     )
			print('First values of logits:'       ,							logits[0, 0, :3, :3]     )
			# assert values
			if model_name == "upernet-swin-tiny":
						lowerCAmelCase__      :      Union[str, Any]         =		torch.tensor(
						    [[-7.5958, -7.5958, -7.4302], [-7.5958, -7.5958, -7.4302], [-7.4797, -7.4797, -7.3068]]     )
			elif model_name == "upernet-swin-small":
						lowerCAmelCase__      :      List[Any]         =		torch.tensor(
						    [[-7.1921, -7.1921, -6.9532], [-7.1921, -7.1921, -6.9532], [-7.0908, -7.0908, -6.8534]]     )
			elif model_name == "upernet-swin-base":
						lowerCAmelCase__      :      Tuple         =		torch.tensor(
						    [[-6.5851, -6.5851, -6.4330], [-6.5851, -6.5851, -6.4330], [-6.4763, -6.4763, -6.3254]]     )
			elif model_name == "upernet-swin-large":
						lowerCAmelCase__      :      List[str]         =		torch.tensor(
						    [[-7.5297, -7.5297, -7.3802], [-7.5297, -7.5297, -7.3802], [-7.4044, -7.4044, -7.2586]]     )
			print('Logits:'       ,							outputs.logits[0, 0, :3, :3]     )
			assert torch.allclose(outputs.logits[0, 0, :3, :3]       ,							SCREAMING_SNAKE_CASE_       ,							atol=1e-4     )
			print('Looks ok!'     )

			if pytorch_dump_folder_path is not None:
						print(F'''Saving model {model_name} to {pytorch_dump_folder_path}'''     )
						model.save_pretrained(SCREAMING_SNAKE_CASE_     )
						print(F'''Saving processor to {pytorch_dump_folder_path}'''     )
						processor.save_pretrained(SCREAMING_SNAKE_CASE_     )

			if push_to_hub:
						print(F'''Pushing model and processor for {model_name} to hub'''     )
						model.push_to_hub(F'''openmmlab/{model_name}'''     )
						processor.push_to_hub(F'''openmmlab/{model_name}'''     )


if __name__ == "__main__":
					lowerCamelCase__          =		argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--model_name""",
					    default="""upernet-swin-tiny""",
					    type=str,
					    choices=[F"""upernet-swin-{size}""" for size in ["""tiny""", """small""", """base""", """large"""]],
					    help="""Name of the Swin + UperNet model you'd like to convert.""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
					)
					parser.add_argument(
					    """--push_to_hub""", action="""store_true""", help="""Whether or not to push the converted model to the 🤗 hub."""
					)

					lowerCamelCase__          =		parser.parse_args()
					convert_upernet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)

import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__          =		get_tests_dir("""fixtures/test_sentencepiece.model""")







@require_sentencepiece
@require_tokenizers
class      A__							(     __magic_name__						,   unittest.TestCase	):
				lowercase			    =  ReformerTokenizer
				lowercase			    =  ReformerTokenizerFast
				lowercase			    =  True
				lowercase			    =  False
				lowercase			    =  True






				def 							_lowerCamelCase		(						self					:      Dict   ):






							'''simple docstring'''







							super().setUp()

							lowerCAmelCase__      :      int         =		ReformerTokenizer(a    ,     keep_accents=a   )
							tokenizer.save_pretrained(self.tmpdirname   )






				def 							_lowerCamelCase		(						self					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		'<s>'
							lowerCAmelCase__      :      Optional[int]         =		1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(a   )    ,     a   )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(a   )    ,     a   )






				def 							_lowerCamelCase		(						self					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Union[str, Any]         =		list(self.get_tokenizer().get_vocab().keys()   )

							self.assertEqual(vocab_keys[0]    ,     '<unk>'   )
							self.assertEqual(vocab_keys[1]    ,     '<s>'   )
							self.assertEqual(vocab_keys[-1]    ,     'j'   )
							self.assertEqual(len(a   )    ,     1_000   )






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							self.assertEqual(self.get_tokenizer().vocab_size    ,     1_000   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							if not self.test_rust_tokenizer:
										return

							lowerCAmelCase__      :      Optional[int]         =		self.get_tokenizer()
							lowerCAmelCase__      :      Any         =		self.get_rust_tokenizer()

							lowerCAmelCase__      :      Optional[int]         =		'I was born in 92000, and this is falsé.'

							lowerCAmelCase__      :      List[Any]         =		tokenizer.tokenize(a   )
							lowerCAmelCase__      :      Optional[int]         =		rust_tokenizer.tokenize(a   )
							self.assertListEqual(a    ,     a   )

							lowerCAmelCase__      :      List[str]         =		tokenizer.encode(a    ,     add_special_tokens=a   )
							lowerCAmelCase__      :      Any         =		rust_tokenizer.encode(a    ,     add_special_tokens=a   )
							self.assertListEqual(a    ,     a   )

							lowerCAmelCase__      :      Any         =		self.get_rust_tokenizer()
							lowerCAmelCase__      :      List[str]         =		tokenizer.encode(a   )
							lowerCAmelCase__      :      Optional[int]         =		rust_tokenizer.encode(a   )
							self.assertListEqual(a    ,     a   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]    ,     a					:      Union[str, Any]=15   ):






							'''simple docstring'''







							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
													lowerCAmelCase__      :      List[str]         =		self.rust_tokenizer_class.from_pretrained(a    ,     **a   )

													# Simple input
													lowerCAmelCase__      :      Any         =		'This is a simple input'
													lowerCAmelCase__      :      str         =		['This is a simple input 1', 'This is a simple input 2']
													lowerCAmelCase__      :      Optional[int]         =		('This is a simple input', 'This is a pair')
													lowerCAmelCase__      :      Optional[int]         =		[
													    ('This is a simple input 1', 'This is a simple input 2'),
													    ('This is a simple pair 1', 'This is a simple pair 2'),
													]

													# Simple input tests
													self.assertRaises(a    ,     tokenizer_r.encode    ,     a    ,     max_length=a    ,     padding='max_length'   )

													# Simple input
													self.assertRaises(a    ,     tokenizer_r.encode_plus    ,     a    ,     max_length=a    ,     padding='max_length'   )

													# Simple input
													self.assertRaises(
													    a    ,     tokenizer_r.batch_encode_plus    ,     a    ,     max_length=a    ,     padding='max_length'    ,     )

													# Pair input
													self.assertRaises(a    ,     tokenizer_r.encode    ,     a    ,     max_length=a    ,     padding='max_length'   )

													# Pair input
													self.assertRaises(a    ,     tokenizer_r.encode_plus    ,     a    ,     max_length=a    ,     padding='max_length'   )

													# Pair input
													self.assertRaises(
													    a    ,     tokenizer_r.batch_encode_plus    ,     a    ,     max_length=a    ,     padding='max_length'    ,     )






				def 							_lowerCamelCase		(						self					:      Optional[int]   ):






							'''simple docstring'''







							pass






				def 							_lowerCamelCase		(						self					:      Optional[int]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[int]         =		ReformerTokenizer(a    ,     keep_accents=a   )

							lowerCAmelCase__      :      Dict         =		tokenizer.tokenize('This is a test'   )
							self.assertListEqual(a    ,     ['▁This', '▁is', '▁a', '▁t', 'est']   )

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(a   )    ,     [285, 46, 10, 170, 382]    ,     )

							lowerCAmelCase__      :      List[str]         =		tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
							self.assertListEqual(
							    a    ,     [
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '9',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        's',
							        'é',
							        '.',
							    ]    ,     )
							lowerCAmelCase__      :      int         =		tokenizer.convert_tokens_to_ids(a   )
							self.assertListEqual(
							    a    ,     [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]    ,     )

							lowerCAmelCase__      :      str         =		tokenizer.convert_ids_to_tokens(a   )
							self.assertListEqual(
							    a    ,     [
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '<unk>',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        's',
							        '<unk>',
							        '.',
							    ]    ,     )






				@cached_property
				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							return ReformerTokenizer.from_pretrained('google/reformer-crime-and-punishment'   )






				@slow
				def 							_lowerCamelCase		(						self					:      Any   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		'Hello World!'
							lowerCAmelCase__      :      int         =		[126, 32, 262, 152, 38, 72, 287]

							self.assertListEqual(a    ,     self.big_tokenizer.encode(a   )   )






				@slow
				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		(
							    'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
							    ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
							)
							lowerCAmelCase__      :      int         =		[
							    108,
							    265,
							    24,
							    111,
							    4,
							    258,
							    156,
							    35,
							    28,
							    275,
							    3,
							    259,
							    297,
							    260,
							    84,
							    4,
							    35,
							    110,
							    44,
							    8,
							    259,
							    91,
							    268,
							    21,
							    11,
							    209,
							    274,
							    109,
							    266,
							    277,
							    117,
							    86,
							    93,
							    315,
							    258,
							    278,
							    258,
							    277,
							    258,
							    0,
							    258,
							    288,
							    258,
							    319,
							    258,
							    0,
							    258,
							    0,
							    258,
							    0,
							    258,
							    0,
							    258,
							    287,
							    258,
							    315,
							    258,
							    289,
							    258,
							    278,
							    99,
							    269,
							    266,
							    262,
							    8,
							    259,
							    241,
							    4,
							    217,
							    230,
							    268,
							    266,
							    55,
							    168,
							    106,
							    75,
							    193,
							    266,
							    223,
							    27,
							    49,
							    26,
							    282,
							    25,
							    264,
							    299,
							    19,
							    26,
							    0,
							    258,
							    277,
							    117,
							    86,
							    93,
							    176,
							    183,
							    270,
							    11,
							    262,
							    42,
							    61,
							    265,
							]

							self.assertListEqual(a    ,     self.big_tokenizer.encode(a   )   )






				@require_torch
				@slow
				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							import torch

							from transformers import ReformerConfig, ReformerModel

							# Build sequence
							lowerCAmelCase__      :      Dict         =		list(self.big_tokenizer.get_vocab().keys()   )[:10]
							lowerCAmelCase__      :      Optional[Any]         =		' '.join(a   )
							lowerCAmelCase__      :      Any         =		self.big_tokenizer.encode_plus(a    ,     return_tensors='pt'   )
							lowerCAmelCase__      :      Optional[int]         =		self.big_tokenizer.batch_encode_plus([sequence, sequence]    ,     return_tensors='pt'   )

							lowerCAmelCase__      :      List[Any]         =		ReformerConfig()
							# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
							lowerCAmelCase__      :      int         =		encoded_sequence['input_ids'].shape
							lowerCAmelCase__      :      Union[str, Any]         =		ReformerModel(a   )

							# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
							assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

							with torch.no_grad():
										model(**a   )
										model(**a   )






				@slow
				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		{'input_ids': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							# This tokenizer does not know some characters like ")".
							# That is the reason why we use very simple texts here.
							# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
							lowerCAmelCase__      :      List[str]         =		[
							    'This is a very simple sentence.',
							    'The quick brown fox jumps over the lazy dog.',
							]

							self.tokenizer_integration_test_util(
							    expected_encoding=a    ,     model_name='google/reformer-crime-and-punishment'    ,     revision='0e6c3decb8211d49bf881013425dc8b0448b3f5a'    ,     padding=a    ,     sequences=a    ,     )