


'''simple docstring'''
import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel




class 			lowerCamelCase_   :






      def __init__(					self				:    List[str]						,						_A				:    List[str]						,						_A				:    Dict=13						,						_A				:    Tuple=7						,						_A				:    Optional[int]=True						,						_A				:    List[Any]=True						,						_A				:    List[str]=False						,						_A				:    str=True						,						_A				:    Optional[Any]=99						,						_A				:    List[str]=32						,						_A				:    int=5						,						_A				:    Dict=4						,						_A				:    Dict=37						,						_A				:    Tuple="gelu"						,						_A				:    int=0.1						,						_A				:    Optional[Any]=0.1						,						_A				:    List[str]=512						,						_A				:    str=16						,						_A				:    Any=2						,						_A				:    Optional[Any]=0.0_2						,						_A				:    Union[str, Any]=3						,						_A				:    str=4						,						_A				:    Union[str, Any]=None						,						):


          '''simple docstring'''


          UpperCAmelCase__			:		str													=				parent
          UpperCAmelCase__			:		Optional[Any]													=				batch_size
          UpperCAmelCase__			:		Optional[int]													=				seq_length
          UpperCAmelCase__			:		List[str]													=				is_training
          UpperCAmelCase__			:		Optional[Any]													=				use_input_mask
          UpperCAmelCase__			:		Any													=				use_token_type_ids
          UpperCAmelCase__			:		int													=				use_labels
          UpperCAmelCase__			:		Optional[Any]													=				vocab_size
          UpperCAmelCase__			:		Optional[int]													=				hidden_size
          UpperCAmelCase__			:		int													=				num_hidden_layers
          UpperCAmelCase__			:		Dict													=				num_attention_heads
          UpperCAmelCase__			:		List[Any]													=				intermediate_size
          UpperCAmelCase__			:		int													=				hidden_act
          UpperCAmelCase__			:		Union[str, Any]													=				hidden_dropout_prob
          UpperCAmelCase__			:		Optional[Any]													=				attention_probs_dropout_prob
          UpperCAmelCase__			:		List[Any]													=				max_position_embeddings
          UpperCAmelCase__			:		Union[str, Any]													=				type_vocab_size
          UpperCAmelCase__			:		Union[str, Any]													=				type_sequence_label_size
          UpperCAmelCase__			:		List[str]													=				initializer_range
          UpperCAmelCase__			:		List[str]													=				num_labels
          UpperCAmelCase__			:		List[str]													=				num_choices
          UpperCAmelCase__			:		Any													=				scope






      def lowercase_							(					self				:    List[str] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Tuple													=				ids_tensor([self.batch_size, self.seq_length]						,						self.vocab_size )

          UpperCAmelCase__			:		List[str]													=				None
          if self.use_input_mask:
              UpperCAmelCase__			:		Optional[int]													=				random_attention_mask([self.batch_size, self.seq_length] )

          UpperCAmelCase__			:		int													=				None
          if self.use_token_type_ids:
              UpperCAmelCase__			:		List[Any]													=				ids_tensor([self.batch_size, self.seq_length]						,						self.type_vocab_size )

          UpperCAmelCase__			:		List[Any]													=				None
          UpperCAmelCase__			:		Tuple													=				None
          UpperCAmelCase__			:		str													=				None
          if self.use_labels:
              UpperCAmelCase__			:		str													=				ids_tensor([self.batch_size]						,						self.type_sequence_label_size )
              UpperCAmelCase__			:		List[str]													=				ids_tensor([self.batch_size, self.seq_length]						,						self.num_labels )
              UpperCAmelCase__			:		str													=				ids_tensor([self.batch_size]						,						self.num_choices )

          UpperCAmelCase__			:		Optional[Any]													=				self.get_config()

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






      def lowercase_							(					self				:    Dict ):


          '''simple docstring'''


          return OpenLlamaConfig(
              vocab_size=self.vocab_size						,						hidden_size=self.hidden_size						,						num_hidden_layers=self.num_hidden_layers						,						num_attention_heads=self.num_attention_heads						,						intermediate_size=self.intermediate_size						,						hidden_act=self.hidden_act						,						hidden_dropout_prob=self.hidden_dropout_prob						,						attention_probs_dropout_prob=self.attention_probs_dropout_prob						,						max_position_embeddings=self.max_position_embeddings						,						type_vocab_size=self.type_vocab_size						,						is_decoder=_A						,						initializer_range=self.initializer_range						,						use_stable_embedding=_A						,						)






      def lowercase_							(					self				:    Any						,						_A				:    Union[str, Any]						,						_A				:    Optional[Any]						,						_A				:    Any						,						_A				:    str						,						_A				:    Dict						,						_A				:    List[Any]						,						_A				:    Any ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[Any]													=				OpenLlamaModel(config=_A )
          model.to(_A )
          model.eval()
          UpperCAmelCase__			:		Any													=				model(_A						,						attention_mask=_A )
          UpperCAmelCase__			:		List[str]													=				model(_A )
          self.parent.assertEqual(result.last_hidden_state.shape						,						(self.batch_size, self.seq_length, self.hidden_size) )






      def lowercase_							(					self				:    int						,						_A				:    Dict						,						_A				:    List[Any]						,						_A				:    Union[str, Any]						,						_A				:    Tuple						,						_A				:    Optional[int]						,						_A				:    int						,						_A				:    Optional[Any]						,						_A				:    Union[str, Any]						,						_A				:    Optional[Any]						,						):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[int]													=				True
          UpperCAmelCase__			:		int													=				OpenLlamaModel(_A )
          model.to(_A )
          model.eval()
          UpperCAmelCase__			:		str													=				model(
              _A						,						attention_mask=_A						,						encoder_hidden_states=_A						,						encoder_attention_mask=_A						,						)
          UpperCAmelCase__			:		str													=				model(
              _A						,						attention_mask=_A						,						encoder_hidden_states=_A						,						)
          UpperCAmelCase__			:		List[str]													=				model(_A						,						attention_mask=_A )
          self.parent.assertEqual(result.last_hidden_state.shape						,						(self.batch_size, self.seq_length, self.hidden_size) )






      def lowercase_							(					self				:    List[Any]						,						_A				:    Optional[int]						,						_A				:    Optional[int]						,						_A				:    str						,						_A				:    Optional[int]						,						_A				:    List[Any]						,						_A				:    str						,						_A				:    Any						,						_A				:    int						,						_A				:    Optional[Any]						,						):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				OpenLlamaForCausalLM(config=_A )
          model.to(_A )
          model.eval()
          UpperCAmelCase__			:		Union[str, Any]													=				model(_A						,						attention_mask=_A						,						labels=_A )
          self.parent.assertEqual(result.logits.shape						,						(self.batch_size, self.seq_length, self.vocab_size) )






      def lowercase_							(					self				:    int						,						_A				:    Dict						,						_A				:    Tuple						,						_A				:    Optional[int]						,						_A				:    Tuple						,						_A				:    int						,						_A				:    Tuple						,						_A				:    Any						,						_A				:    List[str]						,						_A				:    str						,						):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				True
          UpperCAmelCase__			:		List[str]													=				True
          UpperCAmelCase__			:		List[Any]													=				OpenLlamaForCausalLM(config=_A )
          model.to(_A )
          model.eval()

          # first forward pass
          UpperCAmelCase__			:		Any													=				model(
              _A						,						attention_mask=_A						,						encoder_hidden_states=_A						,						encoder_attention_mask=_A						,						use_cache=_A						,						)
          UpperCAmelCase__			:		Any													=				outputs.past_key_values

          # create hypothetical multiple next token and extent to next_input_ids
          UpperCAmelCase__			:		Optional[int]													=				ids_tensor((self.batch_size, 3)						,						config.vocab_size )
          UpperCAmelCase__			:		Any													=				ids_tensor((self.batch_size, 3)						,						vocab_size=2 )

          # append to next input_ids and
          UpperCAmelCase__			:		Union[str, Any]													=				torch.cat([input_ids, next_tokens]						,						dim=-1 )
          UpperCAmelCase__			:		Dict													=				torch.cat([input_mask, next_mask]						,						dim=-1 )

          UpperCAmelCase__			:		Dict													=				model(
              _A						,						attention_mask=_A						,						encoder_hidden_states=_A						,						encoder_attention_mask=_A						,						output_hidden_states=_A						,						)['''hidden_states'''][0]
          UpperCAmelCase__			:		Tuple													=				model(
              _A						,						attention_mask=_A						,						encoder_hidden_states=_A						,						encoder_attention_mask=_A						,						past_key_values=_A						,						output_hidden_states=_A						,						)['''hidden_states'''][0]

          # select random slice
          UpperCAmelCase__			:		int													=				ids_tensor((1,)						,						output_from_past.shape[-1] ).item()
          UpperCAmelCase__			:		Optional[Any]													=				output_from_no_past[:, -3:, random_slice_idx].detach()
          UpperCAmelCase__			:		Dict													=				output_from_past[:, :, random_slice_idx].detach()

          self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1] )

          # test that outputs are equal for slice
          self.parent.assertTrue(torch.allclose(_A						,						_A						,						atol=1e-3 ) )






      def lowercase_							(					self				:    Dict ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[int]													=				self.prepare_config_and_inputs()
          (
              (
              UpperCAmelCase__
          )						,						(
              UpperCAmelCase__
          )						,						(
              UpperCAmelCase__
          )						,						(
              UpperCAmelCase__
          )						,						(
              UpperCAmelCase__
          )						,						(
              UpperCAmelCase__
          )						,						(
              UpperCAmelCase__
          )						,						
          )			:		str													=				config_and_inputs
          UpperCAmelCase__			:		Tuple													=				{'''input_ids''': input_ids, '''attention_mask''': input_mask}
          return config, inputs_dict




@require_torch
class 			lowerCamelCase_   (	__a     ,       __a     ,       __a     ,       unittest.TestCase      ):
      lowerCAmelCase__		     =     (
          (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
      )
      lowerCAmelCase__		     =     (OpenLlamaForCausalLM,) if is_torch_available() else ()
      lowerCAmelCase__		     =     (
          {
              'feature-extraction': OpenLlamaModel,
              'text-classification': OpenLlamaForSequenceClassification,
              'text-generation': OpenLlamaForCausalLM,
              'zero-shot': OpenLlamaForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      lowerCAmelCase__		     =     False
      lowerCAmelCase__		     =     False






      def lowercase_							(					self				:    Union[str, Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Tuple													=				OpenLlamaModelTester(self )
          UpperCAmelCase__			:		List[str]													=				ConfigTester(self						,						config_class=_A						,						hidden_size=37 )






      def lowercase_							(					self				:    Union[str, Any] ):


          '''simple docstring'''


          self.config_tester.run_common_tests()






      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*_A )






      def lowercase_							(					self				:    List[str] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[Any]													=				self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
              UpperCAmelCase__			:		str													=				type
              self.model_tester.create_and_check_model(*_A )






      def lowercase_							(					self				:    Union[str, Any] ):


          '''simple docstring'''


          UpperCAmelCase__						,						UpperCAmelCase__			:		List[Any]													=				self.model_tester.prepare_config_and_inputs_for_common()
          UpperCAmelCase__			:		str													=				3
          UpperCAmelCase__			:		List[str]													=				input_dict['''input_ids''']
          UpperCAmelCase__			:		Optional[int]													=				input_ids.ne(1 ).to(_A )
          UpperCAmelCase__			:		Dict													=				ids_tensor([self.model_tester.batch_size]						,						self.model_tester.type_sequence_label_size )
          UpperCAmelCase__			:		List[str]													=				OpenLlamaForSequenceClassification(_A )
          model.to(_A )
          model.eval()
          UpperCAmelCase__			:		List[Any]													=				model(_A						,						attention_mask=_A						,						labels=_A )
          self.assertEqual(result.logits.shape						,						(self.model_tester.batch_size, self.model_tester.num_labels) )






      def lowercase_							(					self				:    str ):


          '''simple docstring'''


          UpperCAmelCase__						,						UpperCAmelCase__			:		Optional[Any]													=				self.model_tester.prepare_config_and_inputs_for_common()
          UpperCAmelCase__			:		Union[str, Any]													=				3
          UpperCAmelCase__			:		Dict													=				'''single_label_classification'''
          UpperCAmelCase__			:		Union[str, Any]													=				input_dict['''input_ids''']
          UpperCAmelCase__			:		Union[str, Any]													=				input_ids.ne(1 ).to(_A )
          UpperCAmelCase__			:		Union[str, Any]													=				ids_tensor([self.model_tester.batch_size]						,						self.model_tester.type_sequence_label_size )
          UpperCAmelCase__			:		Union[str, Any]													=				OpenLlamaForSequenceClassification(_A )
          model.to(_A )
          model.eval()
          UpperCAmelCase__			:		Dict													=				model(_A						,						attention_mask=_A						,						labels=_A )
          self.assertEqual(result.logits.shape						,						(self.model_tester.batch_size, self.model_tester.num_labels) )






      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          UpperCAmelCase__						,						UpperCAmelCase__			:		Optional[Any]													=				self.model_tester.prepare_config_and_inputs_for_common()
          UpperCAmelCase__			:		Union[str, Any]													=				3
          UpperCAmelCase__			:		str													=				'''multi_label_classification'''
          UpperCAmelCase__			:		List[str]													=				input_dict['''input_ids''']
          UpperCAmelCase__			:		List[str]													=				input_ids.ne(1 ).to(_A )
          UpperCAmelCase__			:		Optional[int]													=				ids_tensor(
              [self.model_tester.batch_size, config.num_labels]						,						self.model_tester.type_sequence_label_size ).to(torch.float )
          UpperCAmelCase__			:		List[str]													=				OpenLlamaForSequenceClassification(_A )
          model.to(_A )
          model.eval()
          UpperCAmelCase__			:		List[str]													=				model(_A						,						attention_mask=_A						,						labels=_A )
          self.assertEqual(result.logits.shape						,						(self.model_tester.batch_size, self.model_tester.num_labels) )






      @unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test''' )
      def lowercase_							(					self				:    Optional[Any] ):


          '''simple docstring'''


          pass






      @parameterized.expand([('''linear''',), ('''dynamic''',)] )
      def lowercase_							(					self				:    List[Any]						,						_A				:    Dict ):


          '''simple docstring'''


          UpperCAmelCase__						,						UpperCAmelCase__			:		Union[str, Any]													=				self.model_tester.prepare_config_and_inputs_for_common()
          UpperCAmelCase__			:		Optional[Any]													=				ids_tensor([1, 10]						,						config.vocab_size )
          UpperCAmelCase__			:		int													=				ids_tensor([1, int(config.max_position_embeddings * 1.5 )]						,						config.vocab_size )

          set_seed(42 )  # Fixed seed at init time so the two models get the same random weights
          UpperCAmelCase__			:		Dict													=				OpenLlamaModel(_A )
          original_model.to(_A )
          original_model.eval()
          UpperCAmelCase__			:		List[str]													=				original_model(_A ).last_hidden_state
          UpperCAmelCase__			:		List[Any]													=				original_model(_A ).last_hidden_state

          set_seed(42 )  # Fixed seed at init time so the two models get the same random weights
          UpperCAmelCase__			:		Tuple													=				{'''type''': scaling_type, '''factor''': 1_0.0}
          UpperCAmelCase__			:		List[Any]													=				OpenLlamaModel(_A )
          scaled_model.to(_A )
          scaled_model.eval()
          UpperCAmelCase__			:		Optional[Any]													=				scaled_model(_A ).last_hidden_state
          UpperCAmelCase__			:		int													=				scaled_model(_A ).last_hidden_state

          # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
          # maximum sequence length, so the outputs for the short input should match.
          if scaling_type == "dynamic":
              self.assertTrue(torch.allclose(_A						,						_A						,						atol=1e-5 ) )
          else:
              self.assertFalse(torch.allclose(_A						,						_A						,						atol=1e-5 ) )

          # The output should be different for long inputs
          self.assertFalse(torch.allclose(_A						,						_A						,						atol=1e-5 ) )




'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__    					=      logging.get_logger(__name__)


UpperCamelCase__    					=      {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all BART models at https://huggingface.co/models?filter=bart
UpperCamelCase__    					=      {
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
}

UpperCamelCase__    					=      {
    '''facebook/bart-base''': 1_0_2_4,
    '''facebook/bart-large''': 1_0_2_4,
    '''facebook/bart-large-mnli''': 1_0_2_4,
    '''facebook/bart-large-cnn''': 1_0_2_4,
    '''facebook/bart-large-xsum''': 1_0_2_4,
    '''yjernite/bart_eli5''': 1_0_2_4,
}





@lru_cache()
def 					a__       (					)    ->	List[Any]:
    UpperCAmelCase__			:		int													=				(
        list(range(ord('''!'''						)  ,					ord('''~'''						) + 1						)						) + list(range(ord('''¡'''						)  ,					ord('''¬'''						) + 1						)						) + list(range(ord('''®'''						)  ,					ord('''ÿ'''						) + 1						)						)
    )
    UpperCAmelCase__			:		Optional[int]													=				bs[:]
    UpperCAmelCase__			:		List[str]													=				0
    for b in range(2**8						):
        if b not in bs:
            bs.append(lowerCAmelCase__						)
            cs.append(2**8 + n						)
            n += 1
    UpperCAmelCase__			:		Any													=				[chr(lowerCAmelCase__						) for n in cs]
    return dict(zip(lowerCAmelCase__  ,					lowerCAmelCase__						)						)





def 					a__       (					lowerCAmelCase__						)    ->	Union[str, Any]:
    UpperCAmelCase__			:		str													=				set()
    UpperCAmelCase__			:		str													=				word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)						)
        UpperCAmelCase__			:		Optional[int]													=				char
    return pairs







class 			lowerCamelCase_   (	__a      ):
      lowerCAmelCase__		     =     VOCAB_FILES_NAMES
      lowerCAmelCase__		     =     PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__		     =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__		     =     ['input_ids', 'attention_mask']






      def __init__(					self				:    Optional[int]						,						_A				:    Optional[int]						,						_A				:    List[Any]						,						_A				:    int="replace"						,						_A				:    List[Any]="<s>"						,						_A				:    List[Any]="</s>"						,						_A				:    List[Any]="</s>"						,						_A				:    Optional[int]="<s>"						,						_A				:    List[str]="<unk>"						,						_A				:    List[str]="<pad>"						,						_A				:    Union[str, Any]="<mask>"						,						_A				:    Any=False						,						**_A				:    Dict						,						):


          '''simple docstring'''


          UpperCAmelCase__			:		Dict													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else bos_token
          UpperCAmelCase__			:		Any													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else eos_token
          UpperCAmelCase__			:		Optional[int]													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else sep_token
          UpperCAmelCase__			:		Tuple													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else cls_token
          UpperCAmelCase__			:		int													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else unk_token
          UpperCAmelCase__			:		Optional[Any]													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else pad_token

          # Mask token behave like a normal word, i.e. include the space before it
          UpperCAmelCase__			:		Optional[int]													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else mask_token

          super().__init__(
              errors=_A						,						bos_token=_A						,						eos_token=_A						,						unk_token=_A						,						sep_token=_A						,						cls_token=_A						,						pad_token=_A						,						mask_token=_A						,						add_prefix_space=_A						,						**_A						,						)

          with open(_A						,						encoding='''utf-8''' ) as vocab_handle:
              UpperCAmelCase__			:		Optional[Any]													=				json.load(_A )
          UpperCAmelCase__			:		Any													=				{v: k for k, v in self.encoder.items()}
          UpperCAmelCase__			:		List[str]													=				errors  # how to handle errors in decoding
          UpperCAmelCase__			:		str													=				bytes_to_unicode()
          UpperCAmelCase__			:		Dict													=				{v: k for k, v in self.byte_encoder.items()}
          with open(_A						,						encoding='''utf-8''' ) as merges_handle:
              UpperCAmelCase__			:		str													=				merges_handle.read().split('''\n''' )[1:-1]
          UpperCAmelCase__			:		Union[str, Any]													=				[tuple(merge.split() ) for merge in bpe_merges]
          UpperCAmelCase__			:		Optional[Any]													=				dict(zip(_A						,						range(len(_A ) ) ) )
          UpperCAmelCase__			:		Optional[int]													=				{}
          UpperCAmelCase__			:		int													=				add_prefix_space

          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          UpperCAmelCase__			:		List[Any]													=				re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''' )






      @property
      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          return len(self.encoder )






      def lowercase_							(					self				:    Tuple ):


          '''simple docstring'''


          return dict(self.encoder						,						**self.added_tokens_encoder )






      def lowercase_							(					self				:    List[Any]						,						_A				:    Tuple ):


          '''simple docstring'''


          if token in self.cache:
              return self.cache[token]
          UpperCAmelCase__			:		Optional[Any]													=				tuple(_A )
          UpperCAmelCase__			:		Dict													=				get_pairs(_A )

          if not pairs:
              return token

          while True:
              UpperCAmelCase__			:		Optional[Any]													=				min(_A						,						key=lambda _A : self.bpe_ranks.get(_A						,						float('''inf''' ) ) )
              if bigram not in self.bpe_ranks:
                  break
              UpperCAmelCase__						,						UpperCAmelCase__			:		str													=				bigram
              UpperCAmelCase__			:		int													=				[]
              UpperCAmelCase__			:		Tuple													=				0
              while i < len(_A ):
                  try:
                      UpperCAmelCase__			:		Optional[int]													=				word.index(_A						,						_A )
                  except ValueError:
                      new_word.extend(word[i:] )
                      break
                  else:
                      new_word.extend(word[i:j] )
                      UpperCAmelCase__			:		Tuple													=				j

                  if word[i] == first and i < len(_A ) - 1 and word[i + 1] == second:
                      new_word.append(first + second )
                      i += 2
                  else:
                      new_word.append(word[i] )
                      i += 1
              UpperCAmelCase__			:		Optional[Any]													=				tuple(_A )
              UpperCAmelCase__			:		List[Any]													=				new_word
              if len(_A ) == 1:
                  break
              else:
                  UpperCAmelCase__			:		Union[str, Any]													=				get_pairs(_A )
          UpperCAmelCase__			:		Optional[Any]													=				''' '''.join(_A )
          UpperCAmelCase__			:		List[Any]													=				word
          return word






      def lowercase_							(					self				:    str						,						_A				:    str ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				[]
          for token in re.findall(self.pat						,						_A ):
              UpperCAmelCase__			:		str													=				''''''.join(
                  self.byte_encoder[b] for b in token.encode('''utf-8''' ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_A ).split(''' ''' ) )
          return bpe_tokens






      def lowercase_							(					self				:    List[str]						,						_A				:    Any ):


          '''simple docstring'''


          return self.encoder.get(_A						,						self.encoder.get(self.unk_token ) )






      def lowercase_							(					self				:    int						,						_A				:    List[str] ):


          '''simple docstring'''


          return self.decoder.get(_A )






      def lowercase_							(					self				:    Tuple						,						_A				:    Any ):


          '''simple docstring'''


          UpperCAmelCase__			:		Any													=				''''''.join(_A )
          UpperCAmelCase__			:		List[str]													=				bytearray([self.byte_decoder[c] for c in text] ).decode('''utf-8'''						,						errors=self.errors )
          return text






      def lowercase_							(					self				:    int						,						_A				:    str						,						_A				:    Optional[str] = None ):


          '''simple docstring'''


          if not os.path.isdir(_A ):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
              return
          UpperCAmelCase__			:		Tuple													=				os.path.join(
              _A						,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
          UpperCAmelCase__			:		Any													=				os.path.join(
              _A						,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''] )

          with open(_A						,						'''w'''						,						encoding='''utf-8''' ) as f:
              f.write(json.dumps(self.encoder						,						indent=2						,						sort_keys=_A						,						ensure_ascii=_A ) + '''\n''' )

          UpperCAmelCase__			:		Union[str, Any]													=				0
          with open(_A						,						'''w'''						,						encoding='''utf-8''' ) as writer:
              writer.write('''#version: 0.2\n''' )
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,						key=lambda _A : kv[1] ):
                  if index != token_index:
                      logger.warning(
                          f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                          ''' Please check that the tokenizer is not corrupted!''' )
                      UpperCAmelCase__			:		List[str]													=				token_index
                  writer.write(''' '''.join(_A ) + '''\n''' )
                  index += 1

          return vocab_file, merge_file






      def lowercase_							(					self				:    str						,						_A				:    List[int]						,						_A				:    Optional[List[int]] = None ):


          '''simple docstring'''


          if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          UpperCAmelCase__			:		List[str]													=				[self.cls_token_id]
          UpperCAmelCase__			:		Union[str, Any]													=				[self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep






      def lowercase_							(					self				:    Optional[int]						,						_A				:    List[int]						,						_A				:    Optional[List[int]] = None						,						_A				:    bool = False ):


          '''simple docstring'''


          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_A						,						token_ids_a=_A						,						already_has_special_tokens=_A )

          if token_ids_a is None:
              return [1] + ([0] * len(_A )) + [1]
          return [1] + ([0] * len(_A )) + [1, 1] + ([0] * len(_A )) + [1]






      def lowercase_							(					self				:    Dict						,						_A				:    List[int]						,						_A				:    Optional[List[int]] = None ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				[self.sep_token_id]
          UpperCAmelCase__			:		Dict													=				[self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep ) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






      def lowercase_							(					self				:    Optional[Any]						,						_A				:    Any						,						_A				:    Dict=False						,						**_A				:    Union[str, Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Any													=				kwargs.pop('''add_prefix_space'''						,						self.add_prefix_space )
          if (is_split_into_words or add_prefix_space) and (len(_A ) > 0 and not text[0].isspace()):
              UpperCAmelCase__			:		Tuple													=				''' ''' + text
          return (text, kwargs)
