




'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_    	=							logging.get_logger(__name__)


lowerCAmelCase_    	=							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
lowerCAmelCase_    	=							{
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
}

lowerCAmelCase_    	=							{
    '''RUCAIBox/mvp''': 1_0_2_4,
}







@lru_cache()
def 	_A			(   ):






   '''simple docstring'''







   A__         =    (
       list(range(ord('!'	)			,ord('~'	) + 1	)	) + list(range(ord('¡'	)			,ord('¬'	) + 1	)	) + list(range(ord('®'	)			,ord('ÿ'	) + 1	)	)
   )
   A__         =    bs[:]
   A__         =    0
   for b in range(2**8	):
      if b not in bs:
         bs.append(UpperCAmelCase	)
         cs.append(2**8 + n	)
         n += 1
   A__         =    [chr(UpperCAmelCase	) for n in cs]
   return dict(zip(UpperCAmelCase			,UpperCAmelCase	)	)







def 	_A			(   UpperCAmelCase	):






   '''simple docstring'''







   A__         =    set()
   A__         =    word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)	)
      A__         =    char
   return pairs







class   _snake_case( UpperCAmelCase			):
 __snake_case:       Optional[Any]									=				VOCAB_FILES_NAMES
 __snake_case:       Dict									=				PRETRAINED_VOCAB_FILES_MAP
 __snake_case:       List[str]									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __snake_case:       Optional[int]									=				['''input_ids''', '''attention_mask''']






 def __init__(self					:     int						,		a					:     Tuple						,		a					:     Any						,		a					:     List[str]="replace"						,		a					:     Any="<s>"						,		a					:     List[str]="</s>"						,		a					:     Optional[int]="</s>"						,		a					:     int="<s>"						,		a					:     int="<unk>"						,		a					:     Dict="<pad>"						,		a					:     Any="<mask>"						,		a					:     Optional[Any]=False						,		**a					:     List[Any]						,		)					->      Union[str, Any]:






    """simple docstring"""


    A__         =    AddedToken(a						,		lstrip=a						,		rstrip=a						) if isinstance(a						,		a						) else bos_token
    A__         =    AddedToken(a						,		lstrip=a						,		rstrip=a						) if isinstance(a						,		a						) else eos_token
    A__         =    AddedToken(a						,		lstrip=a						,		rstrip=a						) if isinstance(a						,		a						) else sep_token
    A__         =    AddedToken(a						,		lstrip=a						,		rstrip=a						) if isinstance(a						,		a						) else cls_token
    A__         =    AddedToken(a						,		lstrip=a						,		rstrip=a						) if isinstance(a						,		a						) else unk_token
    A__         =    AddedToken(a						,		lstrip=a						,		rstrip=a						) if isinstance(a						,		a						) else pad_token

    # Mask token behave like a normal word, i.e. include the space before it
    A__         =    AddedToken(a						,		lstrip=a						,		rstrip=a						) if isinstance(a						,		a						) else mask_token

    super().__init__(
        errors=a						,		bos_token=a						,		eos_token=a						,		unk_token=a						,		sep_token=a						,		cls_token=a						,		pad_token=a						,		mask_token=a						,		add_prefix_space=a						,		**a						,		)
    with open(a						,		encoding='utf-8'						) as vocab_handle:
       A__         =    json.load(a						)
    A__         =    {v: k for k, v in self.encoder.items()}
    A__         =    errors  # how to handle errors in decoding
    A__         =    bytes_to_unicode()
    A__         =    {v: k for k, v in self.byte_encoder.items()}
    with open(a						,		encoding='utf-8'						) as merges_handle:
       A__         =    merges_handle.read().split('\n'						)[1:-1]
    A__         =    [tuple(merge.split()						) for merge in bpe_merges]
    A__         =    dict(zip(a						,		range(len(a						)						)						)						)
    A__         =    {}
    A__         =    add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    A__         =    re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'						)






 @property
 def 	_UpperCamelCase   (self					:     Optional[int]						)					->      Union[str, Any]:






    """simple docstring"""


    return len(self.encoder						)






 def 	_UpperCamelCase   (self					:     List[str]						)					->      str:






    """simple docstring"""


    return dict(self.encoder						,		**self.added_tokens_encoder						)






 def 	_UpperCamelCase   (self					:     List[Any]						,		a					:     Tuple						)					->      List[Any]:






    """simple docstring"""


    if token in self.cache:
       return self.cache[token]
    A__         =    tuple(a						)
    A__         =    get_pairs(a						)

    if not pairs:
       return token

    while True:
       A__         =    min(a						,		key=lambda a						: self.bpe_ranks.get(a						,		float('inf'						)						)						)
       if bigram not in self.bpe_ranks:
          break
       A__     , A__         =    bigram
       A__         =    []
       A__         =    0
       while i < len(a						):
          try:
             A__         =    word.index(a						,		a						)
          except ValueError:
             new_word.extend(word[i:]						)
             break
          else:
             new_word.extend(word[i:j]						)
             A__         =    j

          if word[i] == first and i < len(a						) - 1 and word[i + 1] == second:
             new_word.append(first + second						)
             i += 2
          else:
             new_word.append(word[i]						)
             i += 1
       A__         =    tuple(a						)
       A__         =    new_word
       if len(a						) == 1:
          break
       else:
          A__         =    get_pairs(a						)
    A__         =    ' '.join(a						)
    A__         =    word
    return word






 def 	_UpperCamelCase   (self					:     str						,		a					:     str						)					->      List[str]:






    """simple docstring"""


    A__         =    []
    for token in re.findall(self.pat						,		a						):
       A__         =    ''.join(
           self.byte_encoder[b] for b in token.encode('utf-8'						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a						).split(' '						)						)
    return bpe_tokens






 def 	_UpperCamelCase   (self					:     Union[str, Any]						,		a					:     Tuple						)					->      List[str]:






    """simple docstring"""


    return self.encoder.get(a						,		self.encoder.get(self.unk_token						)						)






 def 	_UpperCamelCase   (self					:     str						,		a					:     Tuple						)					->      List[Any]:






    """simple docstring"""


    return self.decoder.get(a						)






 def 	_UpperCamelCase   (self					:     Optional[int]						,		a					:     List[Any]						)					->      Dict:






    """simple docstring"""


    A__         =    ''.join(a						)
    A__         =    bytearray([self.byte_decoder[c] for c in text]						).decode('utf-8'						,		errors=self.errors						)
    return text






 def 	_UpperCamelCase   (self					:     Optional[Any]						,		a					:     str						,		a					:     Optional[str] = None						)					->      Tuple[str]:






    """simple docstring"""


    if not os.path.isdir(a						):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
       return
    A__         =    os.path.join(
        a						,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
    A__         =    os.path.join(
        a						,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

    with open(a						,		'w'						,		encoding='utf-8'						) as f:
       f.write(json.dumps(self.encoder						,		indent=2						,		sort_keys=a						,		ensure_ascii=a						) + '\n'						)

    A__         =    0
    with open(a						,		'w'						,		encoding='utf-8'						) as writer:
       writer.write('#version: 0.2\n'						)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,		key=lambda a						: kv[1]						):
          if index != token_index:
             logger.warning(
                 f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                 ' Please check that the tokenizer is not corrupted!'						)
             A__         =    token_index
          writer.write(' '.join(a						) + '\n'						)
          index += 1

    return vocab_file, merge_file






 def 	_UpperCamelCase   (self					:     List[Any]						,		a					:     List[int]						,		a					:     Optional[List[int]] = None						)					->      List[int]:






    """simple docstring"""


    if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    A__         =    [self.cls_token_id]
    A__         =    [self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep






 def 	_UpperCamelCase   (self					:     Dict						,		a					:     List[int]						,		a					:     Optional[List[int]] = None						,		a					:     bool = False						)					->      List[int]:






    """simple docstring"""


    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=a						,		token_ids_a=a						,		already_has_special_tokens=a						)

    if token_ids_a is None:
       return [1] + ([0] * len(a						)) + [1]
    return [1] + ([0] * len(a						)) + [1, 1] + ([0] * len(a						)) + [1]






 def 	_UpperCamelCase   (self					:     List[Any]						,		a					:     List[int]						,		a					:     Optional[List[int]] = None						)					->      List[int]:






    """simple docstring"""


    A__         =    [self.sep_token_id]
    A__         =    [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep						) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






 def 	_UpperCamelCase   (self					:     Optional[int]						,		a					:     Optional[int]						,		a					:     str=False						,		**a					:     List[Any]						)					->      Dict:






    """simple docstring"""


    A__         =    kwargs.pop('add_prefix_space'						,		self.add_prefix_space						)
    if (is_split_into_words or add_prefix_space) and (len(a						) > 0 and not text[0].isspace()):
       A__         =    ' ' + text
    return (text, kwargs)








'''simple docstring'''







import hashlib
import importlib.util
import json
import os
import pathlib
from types import ModuleType

import pytest
import requests

lowerCAmelCase_    	=							pathlib.Path.cwd().joinpath('''project_euler''')
lowerCAmelCase_    	=							pathlib.Path.cwd().joinpath(
    '''scripts''', '''project_euler_answers.json'''
)

with open(PROJECT_EULER_ANSWERS_PATH) as file_handle:
    lowerCAmelCase_    	=							json.load(file_handle)







def 	_A			(   UpperCAmelCase	):






   '''simple docstring'''







   A__         =    importlib.util.spec_from_file_location(file_path.name			,str(UpperCAmelCase	)	)
   A__         =    importlib.util.module_from_spec(UpperCAmelCase	)  # type: ignore
   spec.loader.exec_module(UpperCAmelCase	)  # type: ignore
   return module







def 	_A			(   ):






   '''simple docstring'''







   A__         =    []
   for problem_dir_path in PROJECT_EULER_DIR_PATH.iterdir():
      if problem_dir_path.is_file() or problem_dir_path.name.startswith('_'	):
         continue
      for file_path in problem_dir_path.iterdir():
         if file_path.suffix != ".py" or file_path.name.startswith(('_', 'test')	):
            continue
         solution_file_paths.append(UpperCAmelCase	)
   return solution_file_paths







def 	_A			(   ):






   '''simple docstring'''







   with open(os.environ['GITHUB_EVENT_PATH']	) as file:
      A__         =    json.load(UpperCAmelCase	)
   return event["pull_request"]["url"] + "/files"







def 	_A			(   ):






   '''simple docstring'''







   A__         =    []
   A__         =    {
       'Accept': 'application/vnd.github.v3+json',
       'Authorization': 'token ' + os.environ['GITHUB_TOKEN'],
   }
   A__         =    requests.get(get_files_url()			,headers=UpperCAmelCase	).json()
   for file in files:
      A__         =    pathlib.Path.cwd().joinpath(file['filename']	)
      if (
          filepath.suffix != ".py"
          or filepath.name.startswith(('_', 'test')	)
          or not filepath.name.startswith('sol'	)
      ):
         continue
      solution_file_paths.append(UpperCAmelCase	)
   return solution_file_paths







def 	_A			(   ):






   '''simple docstring'''







   if os.environ.get('CI'	) and os.environ.get('GITHUB_EVENT_NAME'	) == "pull_request":
      # Return only if there are any, otherwise default to all solutions
      if filepaths := added_solution_file_path():
         return filepaths
   return all_solution_file_paths()







@pytest.mark.parametrize(
    'solution_path'			,collect_solution_file_paths()			,ids=lambda UpperCAmelCase	: F"""{path.parent.name}/{path.name}"""			,)
def 	_A			(   UpperCAmelCase	):






   '''simple docstring'''







   A__         =    solution_path.parent.name[8:].zfill(3	)
   A__         =    PROBLEM_ANSWERS[problem_number]
   A__         =    convert_path_to_module(UpperCAmelCase	)
   A__         =    str(solution_module.solution()	)  # type: ignore
   A__         =    hashlib.shaaaa(answer.encode()	).hexdigest()
   assert (
       answer == expected
   ), F"""Expected solution to {problem_number} to have hash {expected}, got {answer}"""


