



'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


_UpperCamelCase							 =	logging.get_logger(__name__)


_UpperCamelCase							 =	{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all LED models at https://huggingface.co/models?filter=LED
_UpperCamelCase							 =	{
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

_UpperCamelCase							 =	{
    '''allenai/led-base-16384''': 1_6384,
}


@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def      lowercase_		(      ):

			"""simple docstring"""





			__UpperCAmelCase      :					Dict   =   (
			    list(range(ord("""!"""       )	,							ord("""~"""       ) + 1       )       ) + list(range(ord("""¡"""       )	,							ord("""¬"""       ) + 1       )       ) + list(range(ord("""®"""       )	,							ord("""ÿ"""       ) + 1       )       )
			)
			__UpperCAmelCase      :					Tuple   =   bs[:]
			__UpperCAmelCase      :					int   =   0
			for b in range(2**8       ):
						if b not in bs:
									bs.append(__lowerCAmelCase       )
									cs.append(2**8 + n       )
									n += 1
			__UpperCAmelCase      :					Dict   =   [chr(__lowerCAmelCase       ) for n in cs]
			return dict(zip(__lowerCAmelCase	,							__lowerCAmelCase       )       )


def      lowercase_		(      lowerCAmelCase__		:     List[str]       ):

			"""simple docstring"""





			__UpperCAmelCase      :					Dict   =   set()
			__UpperCAmelCase      :					int   =   word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)       )
						__UpperCAmelCase      :					List[Any]   =   char
			return pairs






class _A     (			UpperCAmelCase__			):
		_SCREAMING_SNAKE_CASE    :			Any  					=	VOCAB_FILES_NAMES
		_SCREAMING_SNAKE_CASE    :			List[str]  					=	PRETRAINED_VOCAB_FILES_MAP
		_SCREAMING_SNAKE_CASE    :			Dict  					=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_SCREAMING_SNAKE_CASE    :			Tuple  					=	["input_ids", "attention_mask"]





		def __init__(   self       ,				__UpperCAmelCase       ,				__UpperCAmelCase       ,				__UpperCAmelCase="replace"       ,				__UpperCAmelCase="<s>"       ,				__UpperCAmelCase="</s>"       ,				__UpperCAmelCase="</s>"       ,				__UpperCAmelCase="<s>"       ,				__UpperCAmelCase="<unk>"       ,				__UpperCAmelCase="<pad>"       ,				__UpperCAmelCase="<mask>"       ,				__UpperCAmelCase=False       ,				**__UpperCAmelCase       ,				)				->   Optional[int]:

					'''simple docstring'''
					__UpperCAmelCase      :					int   =   AddedToken(lowerCamelCase__       ,				lstrip=lowerCamelCase__       ,				rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__       ,				lowerCamelCase__ ) else bos_token
					__UpperCAmelCase      :					Optional[Any]   =   AddedToken(lowerCamelCase__       ,				lstrip=lowerCamelCase__       ,				rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__       ,				lowerCamelCase__ ) else eos_token
					__UpperCAmelCase      :					Tuple   =   AddedToken(lowerCamelCase__       ,				lstrip=lowerCamelCase__       ,				rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__       ,				lowerCamelCase__ ) else sep_token
					__UpperCAmelCase      :					Any   =   AddedToken(lowerCamelCase__       ,				lstrip=lowerCamelCase__       ,				rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__       ,				lowerCamelCase__ ) else cls_token
					__UpperCAmelCase      :					Optional[Any]   =   AddedToken(lowerCamelCase__       ,				lstrip=lowerCamelCase__       ,				rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__       ,				lowerCamelCase__ ) else unk_token
					__UpperCAmelCase      :					int   =   AddedToken(lowerCamelCase__       ,				lstrip=lowerCamelCase__       ,				rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__       ,				lowerCamelCase__ ) else pad_token

					# Mask token behave like a normal word, i.e. include the space before it
					__UpperCAmelCase      :					Any   =   AddedToken(lowerCamelCase__       ,				lstrip=lowerCamelCase__       ,				rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__       ,				lowerCamelCase__ ) else mask_token

					super().__init__(
					    errors=lowerCamelCase__       ,				bos_token=lowerCamelCase__       ,				eos_token=lowerCamelCase__       ,				unk_token=lowerCamelCase__       ,				sep_token=lowerCamelCase__       ,				cls_token=lowerCamelCase__       ,				pad_token=lowerCamelCase__       ,				mask_token=lowerCamelCase__       ,				add_prefix_space=lowerCamelCase__       ,				**lowerCamelCase__       ,				)

					with open(lowerCamelCase__       ,				encoding="""utf-8""" ) as vocab_handle:
								__UpperCAmelCase      :					Tuple   =   json.load(lowerCamelCase__ )
					__UpperCAmelCase      :					List[Any]   =   {v: k for k, v in self.encoder.items()}
					__UpperCAmelCase      :					Optional[int]   =   errors  # how to handle errors in decoding
					__UpperCAmelCase      :					int   =   bytes_to_unicode()
					__UpperCAmelCase      :					Tuple   =   {v: k for k, v in self.byte_encoder.items()}
					with open(lowerCamelCase__       ,				encoding="""utf-8""" ) as merges_handle:
								__UpperCAmelCase      :					Tuple   =   merges_handle.read().split("""\n""" )[1:-1]
					__UpperCAmelCase      :					str   =   [tuple(merge.split() ) for merge in bpe_merges]
					__UpperCAmelCase      :					Dict   =   dict(zip(lowerCamelCase__       ,				range(len(lowerCamelCase__ ) ) ) )
					__UpperCAmelCase      :					List[Any]   =   {}
					__UpperCAmelCase      :					List[Any]   =   add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					__UpperCAmelCase      :					str   =   re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" )





		@property
		# Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
		def 							__A				(   self )				->   Union[str, Any]:

					'''simple docstring'''
					return len(self.encoder )





		def 							__A				(   self )				->   Optional[int]:

					'''simple docstring'''
					return dict(self.encoder       ,				**self.added_tokens_encoder )





		def 							__A				(   self       ,				__UpperCAmelCase )				->   Dict:

					'''simple docstring'''







					if token in self.cache:
								return self.cache[token]
					__UpperCAmelCase      :					Optional[int]   =   tuple(lowerCamelCase__ )
					__UpperCAmelCase      :					int   =   get_pairs(lowerCamelCase__ )

					if not pairs:
								return token

					while True:
								__UpperCAmelCase      :					Union[str, Any]   =   min(lowerCamelCase__       ,				key=lambda __UpperCAmelCase : self.bpe_ranks.get(lowerCamelCase__       ,				float("""inf""" ) ) )
								if bigram not in self.bpe_ranks:
											break
								__UpperCAmelCase      :					List[str]   =   bigram
								__UpperCAmelCase      :					List[str]   =   []
								__UpperCAmelCase      :					Optional[Any]   =   0
								while i < len(lowerCamelCase__ ):
											try:
														__UpperCAmelCase      :					str   =   word.index(lowerCamelCase__       ,				lowerCamelCase__ )
											except ValueError:
														new_word.extend(word[i:] )
														break
											else:
														new_word.extend(word[i:j] )
														__UpperCAmelCase      :					Any   =   j

											if word[i] == first and i < len(lowerCamelCase__ ) - 1 and word[i + 1] == second:
														new_word.append(first + second )
														i += 2
											else:
														new_word.append(word[i] )
														i += 1
								__UpperCAmelCase      :					Any   =   tuple(lowerCamelCase__ )
								__UpperCAmelCase      :					Tuple   =   new_word
								if len(lowerCamelCase__ ) == 1:
											break
								else:
											__UpperCAmelCase      :					Any   =   get_pairs(lowerCamelCase__ )
					__UpperCAmelCase      :					str   =   " ".join(lowerCamelCase__ )
					__UpperCAmelCase      :					int   =   word
					return word





		def 							__A				(   self       ,				__UpperCAmelCase )				->   Dict:

					'''simple docstring'''
					__UpperCAmelCase      :					int   =   []
					for token in re.findall(self.pat       ,				lowerCamelCase__ ):
								__UpperCAmelCase      :					List[str]   =   "".join(
								    self.byte_encoder[b] for b in token.encode("""utf-8""" ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
								bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase__ ).split(""" """ ) )
					return bpe_tokens





		def 							__A				(   self       ,				__UpperCAmelCase )				->   str:

					'''simple docstring'''
					return self.encoder.get(lowerCamelCase__       ,				self.encoder.get(self.unk_token ) )





		def 							__A				(   self       ,				__UpperCAmelCase )				->   List[str]:

					'''simple docstring'''
					return self.decoder.get(lowerCamelCase__ )





		def 							__A				(   self       ,				__UpperCAmelCase )				->   Union[str, Any]:

					'''simple docstring'''
					__UpperCAmelCase      :					List[Any]   =   "".join(lowerCamelCase__ )
					__UpperCAmelCase      :					Dict   =   bytearray([self.byte_decoder[c] for c in text] ).decode("""utf-8"""       ,				errors=self.errors )
					return text





		def 							__A				(   self       ,				__UpperCAmelCase       ,				__UpperCAmelCase = None )				->   Tuple[str]:

					'''simple docstring'''







					if not os.path.isdir(lowerCamelCase__ ):
								logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
								return
					__UpperCAmelCase      :					str   =   os.path.join(
					    lowerCamelCase__       ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )
					__UpperCAmelCase      :					int   =   os.path.join(
					    lowerCamelCase__       ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""] )

					with open(lowerCamelCase__       ,				"""w"""       ,				encoding="""utf-8""" ) as f:
								f.write(json.dumps(self.encoder       ,				indent=2       ,				sort_keys=lowerCamelCase__       ,				ensure_ascii=lowerCamelCase__ ) + """\n""" )

					__UpperCAmelCase      :					List[str]   =   0
					with open(lowerCamelCase__       ,				"""w"""       ,				encoding="""utf-8""" ) as writer:
								writer.write("""#version: 0.2\n""" )
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,				key=lambda __UpperCAmelCase : kv[1] ):
											if index != token_index:
														logger.warning(
														    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
														    """ Please check that the tokenizer is not corrupted!""" )
														__UpperCAmelCase      :					int   =   token_index
											writer.write(""" """.join(lowerCamelCase__ ) + """\n""" )
											index += 1

					return vocab_file, merge_file





		def 							__A				(   self       ,				__UpperCAmelCase       ,				__UpperCAmelCase = None )				->   List[int]:

					'''simple docstring'''







					if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					__UpperCAmelCase      :					int   =   [self.cls_token_id]
					__UpperCAmelCase      :					List[str]   =   [self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep





		def 							__A				(   self       ,				__UpperCAmelCase       ,				__UpperCAmelCase = None       ,				__UpperCAmelCase = False )				->   List[int]:

					'''simple docstring'''







					if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=lowerCamelCase__       ,				token_ids_a=lowerCamelCase__       ,				already_has_special_tokens=lowerCamelCase__ )

					if token_ids_a is None:
								return [1] + ([0] * len(lowerCamelCase__ )) + [1]
					return [1] + ([0] * len(lowerCamelCase__ )) + [1, 1] + ([0] * len(lowerCamelCase__ )) + [1]





		def 							__A				(   self       ,				__UpperCAmelCase       ,				__UpperCAmelCase = None )				->   List[int]:

					'''simple docstring'''
					__UpperCAmelCase      :					Tuple   =   [self.sep_token_id]
					__UpperCAmelCase      :					str   =   [self.cls_token_id]

					if token_ids_a is None:
								return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]





		def 							__A				(   self       ,				__UpperCAmelCase       ,				__UpperCAmelCase=False       ,				**__UpperCAmelCase )				->   List[str]:

					'''simple docstring'''
					__UpperCAmelCase      :					int   =   kwargs.pop("""add_prefix_space"""       ,				self.add_prefix_space )
					if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase__ ) > 0 and not text[0].isspace()):
								__UpperCAmelCase      :					Optional[int]   =   " " + text
					return (text, kwargs)





		def 							__A				(   self       ,				__UpperCAmelCase       ,				__UpperCAmelCase = None       ,				__UpperCAmelCase = PaddingStrategy.DO_NOT_PAD       ,				__UpperCAmelCase = None       ,				__UpperCAmelCase = None       ,				)				->   dict:

					'''simple docstring'''
					__UpperCAmelCase      :					Optional[int]   =   super()._pad(
					    encoded_inputs=lowerCamelCase__       ,				max_length=lowerCamelCase__       ,				padding_strategy=lowerCamelCase__       ,				pad_to_multiple_of=lowerCamelCase__       ,				return_attention_mask=lowerCamelCase__       ,				)

					# Load from model defaults
					if return_attention_mask is None:
								__UpperCAmelCase      :					Tuple   =   "attention_mask" in self.model_input_names

					if return_attention_mask and "global_attention_mask" in encoded_inputs:
								__UpperCAmelCase      :					Union[str, Any]   =   encoded_inputs[self.model_input_names[0]]
								# `global_attention_mask` need to have the same length as other (sequential) inputs.
								__UpperCAmelCase      :					Union[str, Any]   =   len(encoded_inputs["""global_attention_mask"""] ) != len(lowerCamelCase__ )

								if needs_to_be_padded:
											__UpperCAmelCase      :					Optional[Any]   =   len(lowerCamelCase__ ) - len(encoded_inputs["""global_attention_mask"""] )

											if self.padding_side == "right":
														# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
														__UpperCAmelCase      :					str   =   (
														    encoded_inputs["global_attention_mask"] + [-1] * difference
														)
											elif self.padding_side == "left":
														__UpperCAmelCase      :					str   =   [-1] * difference + encoded_inputs[
														    "global_attention_mask"
														]
											else:
														raise ValueError("""Invalid padding strategy:""" + str(self.padding_side ) )

					return encoded_inputs








'''simple docstring'''


def        __lowerCAmelCase       (__lowerCAmelCase = 50      ):
		_UpperCAmelCase						:				Tuple	    =					[[0] * 3 for _ in range(length + 1      )]

		for row_length in range(length + 1      ):
				for tile_length in range(2					,       5      ):
						for tile_start in range(row_length - tile_length + 1      ):
								different_colour_ways_number[row_length][tile_length - 2] += (
								    different_colour_ways_number[row_length - tile_start - tile_length][
								        tile_length - 2
								    ]
								    + 1
								)

		return sum(different_colour_ways_number[length]      )


if __name__ == "__main__":
						print(F'''{solution() = }''')



