


'''simple docstring'''



import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A__		:					Optional[Any]												=  logging.get_logger(__name__)


A__		:					Dict												=  {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all BART models at https://huggingface.co/models?filter=bart
A__		:					Dict												=  {
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
}

A__		:					List[str]												=  {
    'facebook/bart-base': 1024,
    'facebook/bart-large': 1024,
    'facebook/bart-large-mnli': 1024,
    'facebook/bart-large-cnn': 1024,
    'facebook/bart-large-xsum': 1024,
    'yjernite/bart_eli5': 1024,
}



@lru_cache()
def 			UpperCAmelCase__					(	)     ->		int:
     __lowerCamelCase   :	int								=		(
         list(range(ord('!'							)      ,    ord('~'							) + 1							)							) + list(range(ord('¡'							)      ,    ord('¬'							) + 1							)							) + list(range(ord('®'							)      ,    ord('ÿ'							) + 1							)							)
     )
     __lowerCamelCase   :	Dict								=		bs[:]
     __lowerCamelCase   :	Dict								=		0
     for b in range(2**8							):
          if b not in bs:
               bs.append(__UpperCAmelCase							)
               cs.append(2**8 + n							)
               n += 1
     __lowerCamelCase   :	int								=		[chr(__UpperCAmelCase							) for n in cs]
     return dict(zip(__UpperCAmelCase      ,    __UpperCAmelCase							)							)



def 			UpperCAmelCase__					(	UpperCAmelCase_    :				str							)     ->		Optional[int]:
     __lowerCamelCase   :	Optional[Any]								=		set()
     __lowerCamelCase   :	Optional[Any]								=		word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)							)
          __lowerCamelCase   :	int								=		char
     return pairs





class    UpperCAmelCase_  (_SCREAMING_SNAKE_CASE  ):





    """simple docstring"""






    lowerCamelCase  :					Union[str, Any]							     =			VOCAB_FILES_NAMES
    lowerCamelCase  :					str							     =			PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase  :					Tuple							     =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase  :					Tuple							     =			['input_ids', 'attention_mask']







    def __init__(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_="replace"			,     SCREAMING_SNAKE_CASE_="<s>"			,     SCREAMING_SNAKE_CASE_="</s>"			,     SCREAMING_SNAKE_CASE_="</s>"			,     SCREAMING_SNAKE_CASE_="<s>"			,     SCREAMING_SNAKE_CASE_="<unk>"			,     SCREAMING_SNAKE_CASE_="<pad>"			,     SCREAMING_SNAKE_CASE_="<mask>"			,     SCREAMING_SNAKE_CASE_=False			,     **SCREAMING_SNAKE_CASE_			,     )				->     Tuple:
         __lowerCamelCase   :	Optional[Any]								=		AddedToken(_lowerCAmelCase			,     lstrip=_lowerCAmelCase			,     rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase			,     _lowerCAmelCase	) else bos_token
         __lowerCamelCase   :	int								=		AddedToken(_lowerCAmelCase			,     lstrip=_lowerCAmelCase			,     rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase			,     _lowerCAmelCase	) else eos_token
         __lowerCamelCase   :	Union[str, Any]								=		AddedToken(_lowerCAmelCase			,     lstrip=_lowerCAmelCase			,     rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase			,     _lowerCAmelCase	) else sep_token
         __lowerCamelCase   :	Tuple								=		AddedToken(_lowerCAmelCase			,     lstrip=_lowerCAmelCase			,     rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase			,     _lowerCAmelCase	) else cls_token
         __lowerCamelCase   :	Optional[int]								=		AddedToken(_lowerCAmelCase			,     lstrip=_lowerCAmelCase			,     rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase			,     _lowerCAmelCase	) else unk_token
         __lowerCamelCase   :	Any								=		AddedToken(_lowerCAmelCase			,     lstrip=_lowerCAmelCase			,     rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase			,     _lowerCAmelCase	) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         __lowerCamelCase   :	Optional[int]								=		AddedToken(_lowerCAmelCase			,     lstrip=_lowerCAmelCase			,     rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase			,     _lowerCAmelCase	) else mask_token

         super().__init__(
             errors=_lowerCAmelCase			,     bos_token=_lowerCAmelCase			,     eos_token=_lowerCAmelCase			,     unk_token=_lowerCAmelCase			,     sep_token=_lowerCAmelCase			,     cls_token=_lowerCAmelCase			,     pad_token=_lowerCAmelCase			,     mask_token=_lowerCAmelCase			,     add_prefix_space=_lowerCAmelCase			,     **_lowerCAmelCase			,     )

         with open(_lowerCAmelCase			,     encoding='utf-8'	) as vocab_handle:
              __lowerCamelCase   :	Union[str, Any]								=		json.load(_lowerCAmelCase	)
         __lowerCamelCase   :	Dict								=		{v: k for k, v in self.encoder.items()}
         __lowerCamelCase   :	Any								=		errors  # how to handle errors in decoding
         __lowerCamelCase   :	Optional[int]								=		bytes_to_unicode()
         __lowerCamelCase   :	int								=		{v: k for k, v in self.byte_encoder.items()}
         with open(_lowerCAmelCase			,     encoding='utf-8'	) as merges_handle:
              __lowerCamelCase   :	Union[str, Any]								=		merges_handle.read().split('\n'	)[1:-1]
         __lowerCamelCase   :	Optional[int]								=		[tuple(merge.split()	) for merge in bpe_merges]
         __lowerCamelCase   :	int								=		dict(zip(_lowerCAmelCase			,     range(len(_lowerCAmelCase	)	)	)	)
         __lowerCamelCase   :	Optional[int]								=		{}
         __lowerCamelCase   :	List[Any]								=		add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         __lowerCamelCase   :	List[Any]								=		re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'	)







    @property
    def       lowercase_				(							self	)				->     Optional[int]:
         return len(self.encoder	)







    def       lowercase_				(							self	)				->     List[Any]:
         return dict(self.encoder			,     **self.added_tokens_encoder	)







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_	)				->     str:
         if token in self.cache:
              return self.cache[token]
         __lowerCamelCase   :	List[Any]								=		tuple(_lowerCAmelCase	)
         __lowerCamelCase   :	List[Any]								=		get_pairs(_lowerCAmelCase	)

         if not pairs:
              return token

         while True:
              __lowerCamelCase   :	Optional[Any]								=		min(_lowerCAmelCase			,     key=lambda SCREAMING_SNAKE_CASE_	: self.bpe_ranks.get(_lowerCAmelCase			,     float('inf'	)	)	)
              if bigram not in self.bpe_ranks:
                   break
              __lowerCamelCase				,       __lowerCamelCase   :	Any								=		bigram
              __lowerCamelCase   :	int								=		[]
              __lowerCamelCase   :	Optional[int]								=		0
              while i < len(_lowerCAmelCase	):
                   try:
                        __lowerCamelCase   :	Union[str, Any]								=		word.index(_lowerCAmelCase			,     _lowerCAmelCase	)
                   except ValueError:
                        new_word.extend(word[i:]	)
                        break
                   else:
                        new_word.extend(word[i:j]	)
                        __lowerCamelCase   :	List[Any]								=		j

                   if word[i] == first and i < len(_lowerCAmelCase	) - 1 and word[i + 1] == second:
                        new_word.append(first + second	)
                        i += 2
                   else:
                        new_word.append(word[i]	)
                        i += 1
              __lowerCamelCase   :	Dict								=		tuple(_lowerCAmelCase	)
              __lowerCamelCase   :	Union[str, Any]								=		new_word
              if len(_lowerCAmelCase	) == 1:
                   break
              else:
                   __lowerCamelCase   :	Tuple								=		get_pairs(_lowerCAmelCase	)
         __lowerCamelCase   :	Optional[Any]								=		' '.join(_lowerCAmelCase	)
         __lowerCamelCase   :	Tuple								=		word
         return word







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_	)				->     Optional[Any]:
         __lowerCamelCase   :	Union[str, Any]								=		[]
         for token in re.findall(self.pat			,     _lowerCAmelCase	):
              __lowerCamelCase   :	List[str]								=		''.join(
                  self.byte_encoder[b] for b in token.encode('utf-8'	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCAmelCase	).split(' '	)	)
         return bpe_tokens







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_	)				->     int:
         return self.encoder.get(_lowerCAmelCase			,     self.encoder.get(self.unk_token	)	)







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_	)				->     Optional[int]:
         return self.decoder.get(_lowerCAmelCase	)







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_	)				->     str:
         __lowerCamelCase   :	Any								=		''.join(_lowerCAmelCase	)
         __lowerCamelCase   :	Union[str, Any]								=		bytearray([self.byte_decoder[c] for c in text]	).decode('utf-8'			,     errors=self.errors	)
         return text







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None	)				->     Any:
         if not os.path.isdir(_lowerCAmelCase	):
              logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
              return
         __lowerCamelCase   :	Union[str, Any]								=		os.path.join(
             _lowerCAmelCase			,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
         __lowerCamelCase   :	Any								=		os.path.join(
             _lowerCAmelCase			,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

         with open(_lowerCAmelCase			,     'w'			,     encoding='utf-8'	) as f:
              f.write(json.dumps(self.encoder			,     indent=2			,     sort_keys=_lowerCAmelCase			,     ensure_ascii=_lowerCAmelCase	) + '\n'	)

         __lowerCamelCase   :	Union[str, Any]								=		0
         with open(_lowerCAmelCase			,     'w'			,     encoding='utf-8'	) as writer:
              writer.write('#version: 0.2\n'	)
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,     key=lambda SCREAMING_SNAKE_CASE_	: kv[1]	):
                   if index != token_index:
                        logger.warning(
                            f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                            ' Please check that the tokenizer is not corrupted!'	)
                        __lowerCamelCase   :	List[str]								=		token_index
                   writer.write(' '.join(_lowerCAmelCase	) + '\n'	)
                   index += 1

         return vocab_file, merge_file







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None	)				->     Optional[Any]:
         if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __lowerCamelCase   :	Any								=		[self.cls_token_id]
         __lowerCamelCase   :	str								=		[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None			,     SCREAMING_SNAKE_CASE_ = False	)				->     Any:
         if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_lowerCAmelCase			,     token_ids_a=_lowerCAmelCase			,     already_has_special_tokens=_lowerCAmelCase	)

         if token_ids_a is None:
              return [1] + ([0] * len(_lowerCAmelCase	)) + [1]
         return [1] + ([0] * len(_lowerCAmelCase	)) + [1, 1] + ([0] * len(_lowerCAmelCase	)) + [1]







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None	)				->     int:
         __lowerCamelCase   :	Union[str, Any]								=		[self.sep_token_id]
         __lowerCamelCase   :	int								=		[self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep	) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_=False			,     **SCREAMING_SNAKE_CASE_	)				->     List[str]:
         __lowerCamelCase   :	List[str]								=		kwargs.pop('add_prefix_space'			,     self.add_prefix_space	)
         if (is_split_into_words or add_prefix_space) and (len(_lowerCAmelCase	) > 0 and not text[0].isspace()):
              __lowerCamelCase   :	Any								=		' ' + text
         return (text, kwargs)












def 					UpperCAmelCase_							(      __UpperCAmelCase						:	int			,			__UpperCAmelCase						:	int							)					-> int:
      return number | (1 << position)

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	int			,			__UpperCAmelCase						:	int							)					-> int:
      return number & ~(1 << position)

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	int			,			__UpperCAmelCase						:	int							)					-> int:
      return number ^ (1 << position)

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	int			,			__UpperCAmelCase						:	int							)					-> bool:
      return ((number >> position) & 1) == 1

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	int			,			__UpperCAmelCase						:	int							)					-> int:
      return int((number & (1 << position)) != 0							)


if __name__ == "__main__":
   import doctest

   doctest.testmod()