




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__            =     logging.get_logger(__name__)


UpperCamelCase__            =     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCamelCase__            =     {
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
}

UpperCamelCase__            =     {
    """RUCAIBox/mvp""": 1024,
}







@lru_cache()
def       _a (		):
					__lowerCAmelCase							  =			(
					    list(range(ord("!"	) , ord("~"	) + 1	)	) + list(range(ord("¡"	) , ord("¬"	) + 1	)	) + list(range(ord("®"	) , ord("ÿ"	) + 1	)	)
					)
					__lowerCAmelCase							  =			bs[:]
					__lowerCAmelCase							  =			0
					for b in range(2**8	):
										if b not in bs:
															bs.append(UpperCamelCase__	)
															cs.append(2**8 + n	)
															n += 1
					__lowerCAmelCase							  =			[chr(UpperCamelCase__	) for n in cs]
					return dict(zip(UpperCamelCase__ , UpperCamelCase__	)	)







def       _a (		SCREAMING_SNAKE_CASE_				:							List[Any]	):
					__lowerCAmelCase							  =			set()
					__lowerCAmelCase							  =			word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)	)
										__lowerCAmelCase							  =			char
					return pairs


class      a__       (   snake_case__ ):
						_a				:       str   =       VOCAB_FILES_NAMES
						_a				:       Optional[int]   =       PRETRAINED_VOCAB_FILES_MAP
						_a				:       Any   =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a				:       str   =       ["""input_ids""", """attention_mask"""]






						def __init__(  self						,     _A						,     _A						,     _A="replace"						,     _A="<s>"						,     _A="</s>"						,     _A="</s>"						,     _A="<s>"						,     _A="<unk>"						,     _A="<pad>"						,     _A="<mask>"						,     _A=False						,     **_A						,     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			AddedToken(_UpperCAmelCase						,     lstrip=_UpperCAmelCase						,     rstrip=_UpperCAmelCase     ) if isinstance(_UpperCAmelCase						,     _UpperCAmelCase     ) else bos_token
											__lowerCAmelCase							  =			AddedToken(_UpperCAmelCase						,     lstrip=_UpperCAmelCase						,     rstrip=_UpperCAmelCase     ) if isinstance(_UpperCAmelCase						,     _UpperCAmelCase     ) else eos_token
											__lowerCAmelCase							  =			AddedToken(_UpperCAmelCase						,     lstrip=_UpperCAmelCase						,     rstrip=_UpperCAmelCase     ) if isinstance(_UpperCAmelCase						,     _UpperCAmelCase     ) else sep_token
											__lowerCAmelCase							  =			AddedToken(_UpperCAmelCase						,     lstrip=_UpperCAmelCase						,     rstrip=_UpperCAmelCase     ) if isinstance(_UpperCAmelCase						,     _UpperCAmelCase     ) else cls_token
											__lowerCAmelCase							  =			AddedToken(_UpperCAmelCase						,     lstrip=_UpperCAmelCase						,     rstrip=_UpperCAmelCase     ) if isinstance(_UpperCAmelCase						,     _UpperCAmelCase     ) else unk_token
											__lowerCAmelCase							  =			AddedToken(_UpperCAmelCase						,     lstrip=_UpperCAmelCase						,     rstrip=_UpperCAmelCase     ) if isinstance(_UpperCAmelCase						,     _UpperCAmelCase     ) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											__lowerCAmelCase							  =			AddedToken(_UpperCAmelCase						,     lstrip=_UpperCAmelCase						,     rstrip=_UpperCAmelCase     ) if isinstance(_UpperCAmelCase						,     _UpperCAmelCase     ) else mask_token

											super().__init__(
											    errors=_UpperCAmelCase						,     bos_token=_UpperCAmelCase						,     eos_token=_UpperCAmelCase						,     unk_token=_UpperCAmelCase						,     sep_token=_UpperCAmelCase						,     cls_token=_UpperCAmelCase						,     pad_token=_UpperCAmelCase						,     mask_token=_UpperCAmelCase						,     add_prefix_space=_UpperCAmelCase						,     **_UpperCAmelCase						,     )
											with open(_UpperCAmelCase						,     encoding="utf-8"     ) as vocab_handle:
																__lowerCAmelCase							  =			json.load(_UpperCAmelCase     )
											__lowerCAmelCase							  =			{v: k for k, v in self.encoder.items()}
											__lowerCAmelCase							  =			errors  # how to handle errors in decoding
											__lowerCAmelCase							  =			bytes_to_unicode()
											__lowerCAmelCase							  =			{v: k for k, v in self.byte_encoder.items()}
											with open(_UpperCAmelCase						,     encoding="utf-8"     ) as merges_handle:
																__lowerCAmelCase							  =			merges_handle.read().split("\n"     )[1:-1]
											__lowerCAmelCase							  =			[tuple(merge.split()     ) for merge in bpe_merges]
											__lowerCAmelCase							  =			dict(zip(_UpperCAmelCase						,     range(len(_UpperCAmelCase     )     )     )     )
											__lowerCAmelCase							  =			{}
											__lowerCAmelCase							  =			add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											__lowerCAmelCase							  =			re.compile(R"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"     )






						@property
						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											return len(self.encoder     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											return dict(self.encoder						,     **self.added_tokens_encoder     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											if token in self.cache:
																return self.cache[token]
											__lowerCAmelCase							  =			tuple(_UpperCAmelCase     )
											__lowerCAmelCase							  =			get_pairs(_UpperCAmelCase     )

											if not pairs:
																return token

											while True:
																__lowerCAmelCase							  =			min(_UpperCAmelCase						,     key=lambda _A     : self.bpe_ranks.get(_UpperCAmelCase						,     float("inf"     )     )     )
																if bigram not in self.bpe_ranks:
																					break
																__lowerCAmelCase							  =			bigram
																__lowerCAmelCase							  =			[]
																__lowerCAmelCase							  =			0
																while i < len(_UpperCAmelCase     ):
																					try:
																										__lowerCAmelCase							  =			word.index(_UpperCAmelCase						,     _UpperCAmelCase     )
																					except ValueError:
																										new_word.extend(word[i:]     )
																										break
																					else:
																										new_word.extend(word[i:j]     )
																										__lowerCAmelCase							  =			j

																					if word[i] == first and i < len(_UpperCAmelCase     ) - 1 and word[i + 1] == second:
																										new_word.append(first + second     )
																										i += 2
																					else:
																										new_word.append(word[i]     )
																										i += 1
																__lowerCAmelCase							  =			tuple(_UpperCAmelCase     )
																__lowerCAmelCase							  =			new_word
																if len(_UpperCAmelCase     ) == 1:
																					break
																else:
																					__lowerCAmelCase							  =			get_pairs(_UpperCAmelCase     )
											__lowerCAmelCase							  =			''' '''.join(_UpperCAmelCase     )
											__lowerCAmelCase							  =			word
											return word






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			[]
											for token in re.findall(self.pat						,     _UpperCAmelCase     ):
																__lowerCAmelCase							  =			''''''.join(
																    self.byte_encoder[b] for b in token.encode("utf-8"     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_UpperCAmelCase     ).split(" "     )     )
											return bpe_tokens






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											return self.encoder.get(_UpperCAmelCase						,     self.encoder.get(self.unk_token     )     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											return self.decoder.get(_UpperCAmelCase     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			''''''.join(_UpperCAmelCase     )
											__lowerCAmelCase							  =			bytearray([self.byte_decoder[c] for c in text]     ).decode("utf-8"						,     errors=self.errors     )
											return text






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A = None     ):




											"""simple docstring"""



											if not os.path.isdir(_UpperCAmelCase     ):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
																return
											__lowerCAmelCase							  =			os.path.join(
											    _UpperCAmelCase						,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )
											__lowerCAmelCase							  =			os.path.join(
											    _UpperCAmelCase						,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]     )

											with open(_UpperCAmelCase						,     "w"						,     encoding="utf-8"     ) as f:
																f.write(json.dumps(self.encoder						,     indent=2						,     sort_keys=_UpperCAmelCase						,     ensure_ascii=_UpperCAmelCase     ) + "\n"     )

											__lowerCAmelCase							  =			0
											with open(_UpperCAmelCase						,     "w"						,     encoding="utf-8"     ) as writer:
																writer.write("#version: 0.2\n"     )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,     key=lambda _A     : kv[1]     ):
																					if index != token_index:
																										logger.warning(
																										    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																										    " Please check that the tokenizer is not corrupted!"     )
																										__lowerCAmelCase							  =			token_index
																					writer.write(" ".join(_UpperCAmelCase     ) + "\n"     )
																					index += 1

											return vocab_file, merge_file






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A = None     ):




											"""simple docstring"""



											if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											__lowerCAmelCase							  =			[self.cls_token_id]
											__lowerCAmelCase							  =			[self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A = None						,     _A = False     ):




											"""simple docstring"""



											if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_UpperCAmelCase						,     token_ids_a=_UpperCAmelCase						,     already_has_special_tokens=_UpperCAmelCase     )

											if token_ids_a is None:
																return [1] + ([0] * len(_UpperCAmelCase     )) + [1]
											return [1] + ([0] * len(_UpperCAmelCase     )) + [1, 1] + ([0] * len(_UpperCAmelCase     )) + [1]






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A = None     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			[self.sep_token_id]
											__lowerCAmelCase							  =			[self.cls_token_id]

											if token_ids_a is None:
																return len(cls + token_ids_a + sep     ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A=False						,     **_A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			kwargs.pop("add_prefix_space"						,     self.add_prefix_space     )
											if (is_split_into_words or add_prefix_space) and (len(_UpperCAmelCase     ) > 0 and not text[0].isspace()):
																__lowerCAmelCase							  =			''' ''' + text
											return (text, kwargs)











'''simple docstring'''






import timeit


class 	_lowercase    :


   def __init__(			self     , _UpperCAmelCase       ):
       if isinstance(_UpperCAmelCase     , _UpperCAmelCase       ):  # Initializes a matrix identical to the one provided.
           A							:							List[Any]				=    arg
           A							:							Dict				=    len(_UpperCAmelCase       )
       else:  # Initializes a square matrix of the given size and set values to zero.
           A							:							str				=    arg
           A							:							Union[str, Any]				=    [[0 for _ in range(self.n       )] for _ in range(self.n       )]


   def __mul__(			self     , _UpperCAmelCase       ):
       A							:							List[str]				=    Matrix(self.n       )
       for i in range(self.n       ):
           for j in range(self.n       ):
               for k in range(self.n       ):
                   matrix.t[i][j] += self.t[i][k] * b.t[k][j]
       return matrix







def 				_lowerCamelCase(	UpperCamelCase__     :			Union[str, Any]     ,							UpperCamelCase__     :			int		)		->	Union[str, Any]:
    A							:							Tuple				=    Matrix([[1, 0], [0, 1]]		)
    while b > 0:
        if b & 1:
            matrix *= a
        a *= a
        b >>= 1
    return matrix
def 				_lowerCamelCase(	UpperCamelCase__     :			int     ,							UpperCamelCase__     :			Tuple     ,							UpperCamelCase__     :			Any		)		->	Any:
    # Trivial Cases
    if n == 1:
        return fa
    elif n == 2:
        return fa
    A							:							Optional[Any]				=    Matrix([[1, 1], [1, 0]]		)
    A							:							int				=    modular_exponentiation(UpperCamelCase__     ,							n - 2		)
    return fa * matrix.t[0][0] + fa * matrix.t[0][1]
def 				_lowerCamelCase(	UpperCamelCase__     :			str     ,							UpperCamelCase__     :			Optional[Any]     ,							UpperCamelCase__     :			Any		)		->	List[Any]:
    # Trivial Cases
    if n == 1:
        return fa
    elif n == 2:
        return fa

    A							:							str				=    fa
    A							:							List[Any]				=    fa
    n -= 2

    while n > 0:
        A,						A							:							List[str]				=    fn_a + fn_a, fn_a
        n -= 1

    return fn_a
def 				_lowerCamelCase(	)		->	Union[str, Any]:
    A							:							Any				=    '''
from random import randint
from __main__ import fibonacci_with_matrix_exponentiation
'''
    A							:							Optional[int]				=    '''fibonacci_with_matrix_exponentiation(randint(1,70000), 1, 1)'''
    A							:							List[Any]				=    timeit.timeit(setup=UpperCamelCase__     ,							stmt=UpperCamelCase__     ,							number=100		)
    print('''With matrix exponentiation the average execution time is '''     ,							exec_time / 100		)
    return exec_time
def 				_lowerCamelCase(	)		->	List[Any]:
    A							:							str				=    '''
from random import randint
from __main__ import simple_fibonacci
'''
    A							:							Any				=    '''simple_fibonacci(randint(1,70000), 1, 1)'''
    A							:							str				=    timeit.timeit(setup=UpperCamelCase__     ,							stmt=UpperCamelCase__     ,							number=100		)
    print(
        '''Without matrix exponentiation the average execution time is '''     ,							exec_time / 100		)
    return exec_time
def 				_lowerCamelCase(	)		->	Optional[Any]:
    matrix_exponentiation_time()
    simple_fibonacci_time()


if __name__ == "__main__":
     main()
