

from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel







@require_tf
class 					__lowerCamelCase							:
      lowerCamelCase_      : Tuple  			=   MarianConfig
      lowerCamelCase_      : Tuple  			=   {}
      lowerCamelCase_      : str  			=   'gelu'







      def __init__(	self				,    lowerCamelCase				,    lowerCamelCase=13				,    lowerCamelCase=7				,    lowerCamelCase=True				,    lowerCamelCase=False				,    lowerCamelCase=99				,    lowerCamelCase=32				,    lowerCamelCase=2				,    lowerCamelCase=4				,    lowerCamelCase=37				,    lowerCamelCase=0.1				,    lowerCamelCase=0.1				,    lowerCamelCase=20				,    lowerCamelCase=2				,    lowerCamelCase=1				,    lowerCamelCase=0				,    )		->			Optional[int]:
        snake_case_					= parent
        snake_case_					= batch_size
        snake_case_					= seq_length
        snake_case_					= is_training
        snake_case_					= use_labels
        snake_case_					= vocab_size
        snake_case_					= hidden_size
        snake_case_					= num_hidden_layers
        snake_case_					= num_attention_heads
        snake_case_					= intermediate_size

        snake_case_					= hidden_dropout_prob
        snake_case_					= attention_probs_dropout_prob
        snake_case_					= max_position_embeddings
        snake_case_					= eos_token_id
        snake_case_					= pad_token_id
        snake_case_					= bos_token_id







      def       lowerCAmelCase_       (	self  )		->			Optional[int]:
        snake_case_					= ids_tensor([self.batch_size, self.seq_length - 1]				,    self.vocab_size  )
        snake_case_					= tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size  )				,    1  )
        snake_case_					= tf.concat([input_ids, eos_tensor]				,    axis=1  )

        snake_case_					= ids_tensor([self.batch_size, self.seq_length]				,    self.vocab_size  )

        snake_case_					= self.config_cls(
            vocab_size=self.vocab_size				,    d_model=self.hidden_size				,    encoder_layers=self.num_hidden_layers				,    decoder_layers=self.num_hidden_layers				,    encoder_attention_heads=self.num_attention_heads				,    decoder_attention_heads=self.num_attention_heads				,    encoder_ffn_dim=self.intermediate_size				,    decoder_ffn_dim=self.intermediate_size				,    dropout=self.hidden_dropout_prob				,    attention_dropout=self.attention_probs_dropout_prob				,    max_position_embeddings=self.max_position_embeddings				,    eos_token_ids=[2]				,    bos_token_id=self.bos_token_id				,    pad_token_id=self.pad_token_id				,    decoder_start_token_id=self.pad_token_id				,    **self.config_updates				,    )
        snake_case_					= prepare_marian_inputs_dict(lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase  )
        return config, inputs_dict







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase  )		->			List[str]:
        snake_case_					= TFMarianModel(config=lowerCamelCase  ).get_decoder()
        snake_case_					= inputs_dict['input_ids']

        snake_case_					= input_ids[:1, :]
        snake_case_					= inputs_dict['attention_mask'][:1, :]
        snake_case_					= inputs_dict['head_mask']
        snake_case_					= 1

        # first forward pass
        snake_case_					= model(lowerCamelCase				,    attention_mask=lowerCamelCase				,    head_mask=lowerCamelCase				,    use_cache=lowerCamelCase  )

        snake_case_					= outputs.to_tuple()

        # create hypothetical next token and extent to next_input_ids
        snake_case_					= ids_tensor((self.batch_size, 3)				,    config.vocab_size  )
        snake_case_					= tf.cast(ids_tensor((self.batch_size, 3)				,    2  )				,    tf.inta  )

        # append to next input_ids and
        snake_case_					= tf.concat([input_ids, next_tokens]				,    axis=-1  )
        snake_case_					= tf.concat([attention_mask, next_attn_mask]				,    axis=-1  )

        snake_case_					= model(lowerCamelCase				,    attention_mask=lowerCamelCase  )[0]
        snake_case_					= model(lowerCamelCase				,    attention_mask=lowerCamelCase				,    past_key_values=lowerCamelCase  )[0]

        self.parent.assertEqual(next_tokens.shape[1]				,    output_from_past.shape[1]  )

        # select random slice
        snake_case_					= int(ids_tensor((1,)				,    output_from_past.shape[-1]  )  )
        snake_case_					= output_from_no_past[:, -3:, random_slice_idx]
        snake_case_					= output_from_past[:, :, random_slice_idx]

        # test that outputs are equal for slice
        tf.debugging.assert_near(lowerCamelCase				,    lowerCamelCase				,    rtol=1e-3  )



def   UpperCamelCase(   lowercase_     ,   lowercase_     ,   lowercase_     ,   lowercase_=None     ,   lowercase_=None     ,   lowercase_=None     ,   lowercase_=None     ,   lowercase_=None     ,   )      ->							str:





  '''simple docstring'''





  if attention_mask is None:
    snake_case_					= tf.cast(tf.math.not_equal(lowercase_     ,   config.pad_token_id				)     ,   tf.inta				)
  if decoder_attention_mask is None:
    snake_case_					= tf.concat(
        [
            tf.ones(decoder_input_ids[:, :1].shape     ,   dtype=tf.inta				),
            tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]     ,   config.pad_token_id				)     ,   tf.inta				),
        ]     ,   axis=-1     ,   )
  if head_mask is None:
    snake_case_					= tf.ones((config.encoder_layers, config.encoder_attention_heads)				)
  if decoder_head_mask is None:
    snake_case_					= tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
  if cross_attn_head_mask is None:
    snake_case_					= tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
  return {
      "input_ids": input_ids,
      "decoder_input_ids": decoder_input_ids,
      "attention_mask": attention_mask,
      "decoder_attention_mask": decoder_attention_mask,
      "head_mask": head_mask,
      "decoder_head_mask": decoder_head_mask,
      "cross_attn_head_mask": cross_attn_head_mask,
  }







@require_tf
class 					__lowerCamelCase							(      a_    ,				a_    ,				unittest.TestCase      ):
      lowerCamelCase_      : Any  			=   (TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
      lowerCamelCase_      : Union[str, Any]  			=   (TFMarianMTModel,) if is_tf_available() else ()
      lowerCamelCase_      : List[Any]  			=   (
          {
              'conversational': TFMarianMTModel,
              'feature-extraction': TFMarianModel,
              'summarization': TFMarianMTModel,
              'text2text-generation': TFMarianMTModel,
              'translation': TFMarianMTModel,
          }
          if is_tf_available()
          else {}
      )
      lowerCamelCase_      : List[Any]  			=   True
      lowerCamelCase_      : int  			=   False
      lowerCamelCase_      : int  			=   False







      def       lowerCAmelCase_       (	self  )		->			Union[str, Any]:
        snake_case_					= TFMarianModelTester(self  )
        snake_case_					= ConfigTester(self				,    config_class=lowerCamelCase  )







      def       lowerCAmelCase_       (	self  )		->			str:
        self.config_tester.run_common_tests()







      def       lowerCAmelCase_       (	self  )		->			Union[str, Any]:
        snake_case_					= self.model_tester.prepare_config_and_inputs_for_common()
        self.model_tester.check_decoder_model_past_large_inputs(*lowerCamelCase  )







@require_tf
class 					__lowerCamelCase							(      unittest.TestCase      ):
      lowerCamelCase_      : Tuple  			=   1_000  # show more chars for failing integration tests







      @classmethod
      def       lowerCAmelCase_       (	cls  )		->			None:
        snake_case_					= f'''Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}'''
        return cls







      @cached_property
      def       lowerCAmelCase_       (	self  )		->			MarianTokenizer:
        return AutoTokenizer.from_pretrained(self.model_name  )







      @property
      def       lowerCAmelCase_       (	self  )		->			int:
        return self.tokenizer.eos_token_id







      @cached_property
      def       lowerCAmelCase_       (	self  )		->			str:
        warnings.simplefilter("""error"""  )
        snake_case_					= TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name  )
        assert isinstance(lowerCamelCase				,    lowerCamelCase  )
        snake_case_					= model.config
        self.assertListEqual(c.bad_words_ids				,    [[c.pad_token_id]]  )
        self.assertEqual(c.max_length				,    512  )
        self.assertEqual(c.decoder_start_token_id				,    c.pad_token_id  )
        return model







      def       lowerCAmelCase_       (	self				,    **lowerCamelCase  )		->			List[Any]:
        snake_case_					= self.translate_src_text(**lowerCamelCase  )
        self.assertListEqual(self.expected_text				,    lowerCamelCase  )







      def       lowerCAmelCase_       (	self				,    **lowerCamelCase  )		->			int:
        snake_case_					= self.tokenizer(self.src_text				,    **lowerCamelCase				,    padding=lowerCamelCase				,    return_tensors="""tf"""  )
        snake_case_					= self.model.generate(
            model_inputs.input_ids				,    attention_mask=model_inputs.attention_mask				,    num_beams=2				,    max_length=128  )
        snake_case_					= self.tokenizer.batch_decode(generated_ids.numpy()				,    skip_special_tokens=lowerCamelCase  )
        return generated_words







@require_sentencepiece
@require_tokenizers
@require_tf
class 					__lowerCamelCase							(      a_      ):
      lowerCamelCase_      : Optional[int]  			=   'mt'
      lowerCamelCase_      : Any  			=   'en'
      lowerCamelCase_      : Dict  			=   ['Billi messu b\'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem.']
      lowerCamelCase_      : List[Any]  			=   ['Touching gently, Jesus healed a man who was affected by the sad disease of leprosy.']







      @unittest.skip("""Skipping until #12647 is resolved."""  )
      @slow
      def       lowerCAmelCase_       (	self  )		->			Optional[int]:
        self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 					__lowerCamelCase							(      a_      ):
      lowerCamelCase_      : Optional[Any]  			=   'en'
      lowerCamelCase_      : List[str]  			=   'zh'
      lowerCamelCase_      : str  			=   ['My name is Wolfgang and I live in Berlin']
      lowerCamelCase_      : Union[str, Any]  			=   ['我叫沃尔夫冈 我住在柏林']







      @unittest.skip("""Skipping until #12647 is resolved."""  )
      @slow
      def       lowerCAmelCase_       (	self  )		->			str:
        self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 					__lowerCamelCase							(      a_      ):
      lowerCamelCase_      : Optional[int]  			=   'en'
      lowerCamelCase_      : Any  			=   'ROMANCE'
      lowerCamelCase_      : str  			=   [
          '>>fr<< Don\'t spend so much time watching TV.',
          '>>pt<< Your message has been sent.',
          '>>es<< He\'s two years older than me.',
      ]
      lowerCamelCase_      : List[Any]  			=   [
          'Ne passez pas autant de temps à regarder la télé.',
          'A sua mensagem foi enviada.',
          'Es dos años más viejo que yo.',
      ]







      @unittest.skip("""Skipping until #12647 is resolved."""  )
      @slow
      def       lowerCAmelCase_       (	self  )		->			Optional[int]:
        self._assert_generated_batch_equal_expected()







      @unittest.skip("""Skipping until #12647 is resolved."""  )
      @slow
      def       lowerCAmelCase_       (	self  )		->			Any:
        snake_case_					= TranslationPipeline(self.model				,    self.tokenizer				,    framework="""tf"""  )
        snake_case_					= pipeline(self.src_text  )
        self.assertEqual(self.expected_text				,    [x["""translation_text"""] for x in output]  )
import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
 if is_torch_available():
  import torch
 if is_tf_available():
  import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


UpperCAmelCase      		=					logging.get_logger(__name__)

UpperCAmelCase      		=					{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

UpperCAmelCase      		=					{
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

UpperCAmelCase      		=					{
    '''Salesforce/codegen-350M-mono''': 20_48,
}


@lru_cache()
def SCREAMING_SNAKE_CASE					(  ):
       _UpperCamelCase  :							Union[str, Any]	    =	(
           list(range(ord('!'					)		,    ord('~'					) + 1					)					) + list(range(ord('¡'					)		,    ord('¬'					) + 1					)					) + list(range(ord('®'					)		,    ord('ÿ'					) + 1					)					)
       )
       _UpperCamelCase  :							List[Any]	    =	bs[:]
       _UpperCamelCase  :							Optional[Any]	    =	0
       for b in range(2**8					):
              if b not in bs:
                     bs.append(lowercase_					)
                     cs.append(2**8 + n					)
                     n += 1
       _UpperCamelCase  :							int	    =	[chr(lowercase_					) for n in cs]
       return dict(zip(lowercase_		,    lowercase_					)					)


def SCREAMING_SNAKE_CASE					(  lowercase_:						Optional[int]					):
       _UpperCamelCase  :							Any	    =	set()
       _UpperCamelCase  :							str	    =	word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)					)
              _UpperCamelCase  :							Optional[int]	    =	char
       return pairs





class  __lowerCAmelCase	(							a_    ):



 '''simple docstring'''
 _lowerCamelCase    :						int					     =			VOCAB_FILES_NAMES
 _lowerCamelCase    :						str					     =			PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase    :						str					     =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase    :						Dict					     =			['input_ids', 'attention_mask']

 def __init__(      self							:		List[str]							,      lowerCamelCase							:		List[str]							,      lowerCamelCase							:		Tuple							,      lowerCamelCase							:		Union[str, Any]="replace"							,      lowerCamelCase							:		List[str]="<|endoftext|>"							,      lowerCamelCase							:		Union[str, Any]="<|endoftext|>"							,      lowerCamelCase							:		int="<|endoftext|>"							,      lowerCamelCase							:		str=None							,      lowerCamelCase							:		int=False							,      lowerCamelCase							:		Optional[int]=False							,      **lowerCamelCase							:		List[str]							,      )			->				Any:
        _UpperCamelCase  :							List[Any]	    =	AddedToken(lowerCamelCase							,      lstrip=lowerCamelCase							,      rstrip=lowerCamelCase	) if isinstance(lowerCamelCase							,      lowerCamelCase	) else bos_token
        _UpperCamelCase  :							Union[str, Any]	    =	AddedToken(lowerCamelCase							,      lstrip=lowerCamelCase							,      rstrip=lowerCamelCase	) if isinstance(lowerCamelCase							,      lowerCamelCase	) else eos_token
        _UpperCamelCase  :							str	    =	AddedToken(lowerCamelCase							,      lstrip=lowerCamelCase							,      rstrip=lowerCamelCase	) if isinstance(lowerCamelCase							,      lowerCamelCase	) else unk_token
        _UpperCamelCase  :							List[str]	    =	AddedToken(lowerCamelCase							,      lstrip=lowerCamelCase							,      rstrip=lowerCamelCase	) if isinstance(lowerCamelCase							,      lowerCamelCase	) else pad_token
        super().__init__(
            errors=lowerCamelCase							,      unk_token=lowerCamelCase							,      bos_token=lowerCamelCase							,      eos_token=lowerCamelCase							,      pad_token=lowerCamelCase							,      add_prefix_space=lowerCamelCase							,      add_bos_token=lowerCamelCase							,      **lowerCamelCase							,      )
        _UpperCamelCase  :							str	    =	add_bos_token

        with open(lowerCamelCase							,      encoding='utf-8'	) as vocab_handle:
               _UpperCamelCase  :							Tuple	    =	json.load(lowerCamelCase	)
        _UpperCamelCase  :							List[str]	    =	{v: k for k, v in self.encoder.items()}
        _UpperCamelCase  :							Any	    =	errors  # how to handle errors in decoding
        _UpperCamelCase  :							int	    =	bytes_to_unicode()
        _UpperCamelCase  :							Union[str, Any]	    =	{v: k for k, v in self.byte_encoder.items()}
        with open(lowerCamelCase							,      encoding='utf-8'	) as merges_handle:
               _UpperCamelCase  :							Optional[Any]	    =	merges_handle.read().split('\n'	)[1:-1]
        _UpperCamelCase  :							Optional[Any]	    =	[tuple(merge.split()	) for merge in bpe_merges]
        _UpperCamelCase  :							List[str]	    =	dict(zip(lowerCamelCase							,      range(len(lowerCamelCase	)	)	)	)
        _UpperCamelCase  :							Tuple	    =	{}
        _UpperCamelCase  :							Union[str, Any]	    =	add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        _UpperCamelCase  :							Union[str, Any]	    =	re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'	)

 @property
 def        __snake_case       (      self							:		Union[str, Any]	)			->				Any:
        return len(self.encoder	)

 def        __snake_case       (      self							:		Dict	)			->				int:
        return dict(self.encoder							,      **self.added_tokens_encoder	)

 def        __snake_case       (      self							:		List[Any]							,      lowerCamelCase							:		List[str]	)			->				Dict:
        if token in self.cache:
               return self.cache[token]
        _UpperCamelCase  :							List[str]	    =	tuple(lowerCamelCase	)
        _UpperCamelCase  :							Optional[Any]	    =	get_pairs(lowerCamelCase	)

        if not pairs:
               return token

        while True:
               _UpperCamelCase  :							Optional[int]	    =	min(lowerCamelCase							,      key=lambda lowerCamelCase	: self.bpe_ranks.get(lowerCamelCase							,      float('inf'	)	)	)
               if bigram not in self.bpe_ranks:
                      break
               _UpperCamelCase    ,   _UpperCamelCase  :							List[Any]	    =	bigram
               _UpperCamelCase  :							Optional[Any]	    =	[]
               _UpperCamelCase  :							str	    =	0
               while i < len(lowerCamelCase	):
                      try:
                             _UpperCamelCase  :							Optional[int]	    =	word.index(lowerCamelCase							,      lowerCamelCase	)
                      except ValueError:
                             new_word.extend(word[i:]	)
                             break
                      else:
                             new_word.extend(word[i:j]	)
                             _UpperCamelCase  :							Tuple	    =	j

                      if word[i] == first and i < len(lowerCamelCase	) - 1 and word[i + 1] == second:
                             new_word.append(first + second	)
                             i += 2
                      else:
                             new_word.append(word[i]	)
                             i += 1
               _UpperCamelCase  :							int	    =	tuple(lowerCamelCase	)
               _UpperCamelCase  :							List[Any]	    =	new_word
               if len(lowerCamelCase	) == 1:
                      break
               else:
                      _UpperCamelCase  :							Any	    =	get_pairs(lowerCamelCase	)
        _UpperCamelCase  :							Dict	    =	' '.join(lowerCamelCase	)
        _UpperCamelCase  :							int	    =	word
        return word

 def        __snake_case       (      self							:		str							,      lowerCamelCase							:		Dict							,      lowerCamelCase							:		Tuple=None	)			->				Dict:
        if self.add_bos_token:
               _UpperCamelCase  :							Dict	    =	[self.bos_token_id]
        else:
               _UpperCamelCase  :							Union[str, Any]	    =	[]

        _UpperCamelCase  :							str	    =	bos_token_ids + token_ids_a

        if token_ids_a is None:
               return output

        return output + bos_token_ids + token_ids_a

 def        __snake_case       (      self							:		int							,      lowerCamelCase							:		int	)			->				Optional[Any]:
        _UpperCamelCase  :							List[str]	    =	[]
        for token in re.findall(self.pat							,      lowerCamelCase	):
               _UpperCamelCase  :							Optional[int]	    =	''.join(
                   self.byte_encoder[b] for b in token.encode('utf-8'	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase	).split(' '	)	)
        return bpe_tokens

 def        __snake_case       (      self							:		Any							,      lowerCamelCase							:		Union[str, Any]	)			->				Any:
        return self.encoder.get(lowerCamelCase							,      self.encoder.get(self.unk_token	)	)

 def        __snake_case       (      self							:		int							,      lowerCamelCase							:		List[Any]	)			->				Union[str, Any]:
        return self.decoder.get(lowerCamelCase	)

 def        __snake_case       (      self							:		List[str]							,      lowerCamelCase							:		Any	)			->				Optional[Any]:
        _UpperCamelCase  :							int	    =	''.join(lowerCamelCase	)
        _UpperCamelCase  :							Union[str, Any]	    =	bytearray([self.byte_decoder[c] for c in text]	).decode('utf-8'							,      errors=self.errors	)
        return text

 def        __snake_case       (      self							:		Dict							,      lowerCamelCase							:		str							,      lowerCamelCase							:		Optional[str] = None	)			->				Tuple[str]:
        if not os.path.isdir(lowerCamelCase	):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"	)
               return
        _UpperCamelCase  :							str	    =	os.path.join(
            lowerCamelCase							,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
        _UpperCamelCase  :							Optional[Any]	    =	os.path.join(
            lowerCamelCase							,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

        with open(lowerCamelCase							,      'w'							,      encoding='utf-8'	) as f:
               f.write(json.dumps(self.encoder							,      indent=2							,      sort_keys=lowerCamelCase							,      ensure_ascii=lowerCamelCase	) + '\n'	)

        _UpperCamelCase  :							Optional[Any]	    =	0
        with open(lowerCamelCase							,      'w'							,      encoding='utf-8'	) as writer:
               writer.write('#version: 0.2\n'	)
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,      key=lambda lowerCamelCase	: kv[1]	):
                      if index != token_index:
                             logger.warning(
                                 f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                                 ' Please check that the tokenizer is not corrupted!'	)
                             _UpperCamelCase  :							str	    =	token_index
                      writer.write(' '.join(lowerCamelCase	) + '\n'	)
                      index += 1

        return vocab_file, merge_file

 def        __snake_case       (      self							:		List[str]							,      lowerCamelCase							:		Union[str, Any]							,      lowerCamelCase							:		int=False							,      **lowerCamelCase							:		int	)			->				List[Any]:
        _UpperCamelCase  :							Dict	    =	kwargs.pop('add_prefix_space'							,      self.add_prefix_space	)
        if is_split_into_words or add_prefix_space:
               _UpperCamelCase  :							Any	    =	' ' + text
        return (text, kwargs)

 def        __snake_case       (      self							:		str							,      lowerCamelCase							:		Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]							,      lowerCamelCase							:		bool = False							,      lowerCamelCase							:		bool = None							,      lowerCamelCase							:		Optional[List[str]] = None							,      **lowerCamelCase							:		List[str]							,      )			->				str:
        _UpperCamelCase  :							Optional[int]	    =	super()._decode(
            token_ids=lowerCamelCase							,      skip_special_tokens=lowerCamelCase							,      clean_up_tokenization_spaces=lowerCamelCase							,      **lowerCamelCase							,      )

        if truncate_before_pattern is not None and len(lowerCamelCase	) > 0:
               _UpperCamelCase  :							Optional[Any]	    =	self.truncate(lowerCamelCase							,      lowerCamelCase	)

        return decoded_text

 def        __snake_case       (      self							:		Optional[int]							,      lowerCamelCase							:		Union[str, Any]							,      lowerCamelCase							:		Any	)			->				Optional[int]:
        def find_re(lowerCamelCase							:		Any							,      lowerCamelCase							:		List[Any]							,      lowerCamelCase							:		str	):
               _UpperCamelCase  :							Optional[int]	    =	pattern.search(lowerCamelCase							,      lowerCamelCase	)
               return m.start() if m else -1

        _UpperCamelCase  :							Dict	    =	[re.compile(lowerCamelCase							,      re.MULTILINE	) for pattern in truncate_before_pattern]

        _UpperCamelCase  :							Union[str, Any]	    =	list(re.finditer('^print'							,      lowerCamelCase							,      re.MULTILINE	)	)

        if len(lowerCamelCase	) > 1:
               _UpperCamelCase  :							Tuple	    =	completion[: prints[1].start()]

        _UpperCamelCase  :							Union[str, Any]	    =	list(re.finditer('^def'							,      lowerCamelCase							,      re.MULTILINE	)	)

        if len(lowerCamelCase	) > 1:
               _UpperCamelCase  :							List[str]	    =	completion[: defs[1].start()]

        _UpperCamelCase  :							Tuple	    =	0

        _UpperCamelCase  :							List[str]	    =	[
            pos for pos in [find_re(lowerCamelCase							,      lowerCamelCase							,      lowerCamelCase	) for terminal in terminals] if pos != -1
        ]

        if len(lowerCamelCase	) > 0:
               return completion[: min(lowerCamelCase	)]
        else:
               return completion



