code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  from __future__ import annotations
  
  import unittest
  
  from transformers import MPNetConfig, is_tf_available
  from transformers.testing_utils import require_tf, slow
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_tf_available():
         import tensorflow as tf
  
         from transformers.models.mpnet.modeling_tf_mpnet import (
             TFMPNetForMaskedLM,
             TFMPNetForMultipleChoice,
             TFMPNetForQuestionAnswering,
             TFMPNetForSequenceClassification,
             TFMPNetForTokenClassification,
             TFMPNetModel,
         )
  
  
  
  
  
  
  class 						lowercase     :
  
  
  
  
  
  
        def __init__(self ,  a__ ,  a__=1_3 ,  a__=7 ,  a__=True ,  a__=True ,  a__=False ,  a__=True ,  a__=9_9 ,  a__=6_4 ,  a__=2 ,  a__=4 ,  a__=6_4 ,  a__="gelu" ,  a__=0.1 ,  a__=0.1 ,  a__=5_1_2 ,  a__=1_6 ,  a__=2 ,  a__=0.02 ,  a__=3 ,  a__=4 ,  a__=None ,  )       ->   List[str]:
           SCREAMING_SNAKE_CASE__			:				int       						= parent
           SCREAMING_SNAKE_CASE__			:				Tuple       						= batch_size
           SCREAMING_SNAKE_CASE__			:				Any       						= seq_length
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= is_training
           SCREAMING_SNAKE_CASE__			:				List[str]       						= use_input_mask
           SCREAMING_SNAKE_CASE__			:				Dict       						= use_token_type_ids
           SCREAMING_SNAKE_CASE__			:				Dict       						= use_labels
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= vocab_size
           SCREAMING_SNAKE_CASE__			:				Dict       						= hidden_size
           SCREAMING_SNAKE_CASE__			:				Tuple       						= num_hidden_layers
           SCREAMING_SNAKE_CASE__			:				str       						= num_attention_heads
           SCREAMING_SNAKE_CASE__			:				Any       						= intermediate_size
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= hidden_act
           SCREAMING_SNAKE_CASE__			:				Dict       						= hidden_dropout_prob
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= attention_probs_dropout_prob
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= max_position_embeddings
           SCREAMING_SNAKE_CASE__			:				int       						= type_vocab_size
           SCREAMING_SNAKE_CASE__			:				str       						= type_sequence_label_size
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= initializer_range
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= num_labels
           SCREAMING_SNAKE_CASE__			:				List[str]       						= num_choices
           SCREAMING_SNAKE_CASE__			:				Any       						= scope
  
  
  
  
  
  
        def    A__(self					)       ->   str:
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size					)
  
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= None
           if self.use_input_mask:
              SCREAMING_SNAKE_CASE__			:				List[str]       						= random_attention_mask([self.batch_size, self.seq_length]					)
  
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= None
           SCREAMING_SNAKE_CASE__			:				int       						= None
           SCREAMING_SNAKE_CASE__			:				Any       						= None
           if self.use_labels:
              SCREAMING_SNAKE_CASE__			:				Dict       						= ids_tensor([self.batch_size] ,  self.type_sequence_label_size					)
              SCREAMING_SNAKE_CASE__			:				Dict       						= ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels					)
              SCREAMING_SNAKE_CASE__			:				List[str]       						= ids_tensor([self.batch_size] ,  self.num_choices					)
  
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= MPNetConfig(
               vocab_size=self.vocab_size ,  hidden_size=self.hidden_size ,  num_hidden_layers=self.num_hidden_layers ,  num_attention_heads=self.num_attention_heads ,  intermediate_size=self.intermediate_size ,  hidden_act=self.hidden_act ,  hidden_dropout_prob=self.hidden_dropout_prob ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob ,  max_position_embeddings=self.max_position_embeddings ,  initializer_range=self.initializer_range ,  )
           return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Any:
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= TFMPNetModel(config=a__					)
           SCREAMING_SNAKE_CASE__			:				int       						= {'''input_ids''': input_ids, '''attention_mask''': input_mask}
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= model(a__					)
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [input_ids, input_mask]
           SCREAMING_SNAKE_CASE__			:				str       						= model(a__					)
           self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Optional[int]:
           SCREAMING_SNAKE_CASE__			:				List[str]       						= TFMPNetForMaskedLM(config=a__					)
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= {'''input_ids''': input_ids, '''attention_mask''': input_mask}
           SCREAMING_SNAKE_CASE__			:				int       						= model(a__					)
           self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size)					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   List[str]:
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= TFMPNetForQuestionAnswering(config=a__					)
           SCREAMING_SNAKE_CASE__			:				Any       						= {
               '''input_ids''': input_ids,
               '''attention_mask''': input_mask,
           }
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= model(a__					)
           self.parent.assertEqual(result.start_logits.shape ,  (self.batch_size, self.seq_length)					)
           self.parent.assertEqual(result.end_logits.shape ,  (self.batch_size, self.seq_length)					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Any:
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= self.num_labels
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= TFMPNetForSequenceClassification(a__					)
           SCREAMING_SNAKE_CASE__			:				List[str]       						= {'''input_ids''': input_ids, '''attention_mask''': input_mask}
           SCREAMING_SNAKE_CASE__			:				str       						= model(a__					)
           self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_labels)					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Optional[Any]:
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= self.num_choices
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= TFMPNetForMultipleChoice(a__					)
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= tf.tile(tf.expand_dims(a__ ,  1					) ,  (1, self.num_choices, 1)					)
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= tf.tile(tf.expand_dims(a__ ,  1					) ,  (1, self.num_choices, 1)					)
           SCREAMING_SNAKE_CASE__			:				int       						= {
               '''input_ids''': multiple_choice_inputs_ids,
               '''attention_mask''': multiple_choice_input_mask,
           }
           SCREAMING_SNAKE_CASE__			:				Tuple       						= model(a__					)
           self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_choices)					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Optional[Any]:
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= self.num_labels
           SCREAMING_SNAKE_CASE__			:				Tuple       						= TFMPNetForTokenClassification(a__					)
           SCREAMING_SNAKE_CASE__			:				Tuple       						= {'''input_ids''': input_ids, '''attention_mask''': input_mask}
           SCREAMING_SNAKE_CASE__			:				Tuple       						= model(a__					)
           self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.num_labels)					)
  
  
  
  
  
  
        def    A__(self					)       ->   Optional[Any]:
           SCREAMING_SNAKE_CASE__			:				Tuple       						= self.prepare_config_and_inputs()
           (SCREAMING_SNAKE_CASE__)			:				Optional[Any]       						= config_and_inputs
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= {'''input_ids''': input_ids, '''attention_mask''': input_mask}
           return config, inputs_dict
  
  
  
  
  
  
  @require_tf
  class 						lowercase     (   UpperCAmelCase_				,			UpperCAmelCase_				,			unittest.TestCase):
        __lowercase 		=  (
            (
                TFMPNetForMaskedLM,
                TFMPNetForMultipleChoice,
                TFMPNetForQuestionAnswering,
                TFMPNetForSequenceClassification,
                TFMPNetForTokenClassification,
                TFMPNetModel,
            )
            if is_tf_available()
            else ()
        )
        __lowercase 		=  (
            {
                '''feature-extraction''': TFMPNetModel,
                '''fill-mask''': TFMPNetForMaskedLM,
                '''question-answering''': TFMPNetForQuestionAnswering,
                '''text-classification''': TFMPNetForSequenceClassification,
                '''token-classification''': TFMPNetForTokenClassification,
                '''zero-shot''': TFMPNetForSequenceClassification,
            }
            if is_tf_available()
            else {}
        )
        __lowercase 		=  False
        __lowercase 		=  False
  
  
  
  
  
  
        def    A__(self					)       ->   Optional[Any]:
           SCREAMING_SNAKE_CASE__			:				int       						= TFMPNetModelTester(self					)
           SCREAMING_SNAKE_CASE__			:				List[str]       						= ConfigTester(self ,  config_class=a__ ,  hidden_size=3_7					)
  
  
  
  
  
  
        def    A__(self					)       ->   Optional[Any]:
           self.config_tester.run_common_tests()
  
  
  
  
  
  
        def    A__(self					)       ->   Dict:
           SCREAMING_SNAKE_CASE__			:				str       						= self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_model(*a__					)
  
  
  
  
  
  
        def    A__(self					)       ->   Tuple:
           SCREAMING_SNAKE_CASE__			:				List[str]       						= self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_masked_lm(*a__					)
  
  
  
  
  
  
        def    A__(self					)       ->   Any:
           SCREAMING_SNAKE_CASE__			:				str       						= self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_question_answering(*a__					)
  
  
  
  
  
  
        def    A__(self					)       ->   Dict:
           SCREAMING_SNAKE_CASE__			:				str       						= self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_sequence_classification(*a__					)
  
  
  
  
  
  
        def    A__(self					)       ->   Any:
           SCREAMING_SNAKE_CASE__			:				str       						= self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_multiple_choice(*a__					)
  
  
  
  
  
  
        def    A__(self					)       ->   int:
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_token_classification(*a__					)
  
  
  
  
  
  
        @slow
        def    A__(self					)       ->   Optional[int]:
           for model_name in ["microsoft/mpnet-base"]:
              SCREAMING_SNAKE_CASE__			:				Dict       						= TFMPNetModel.from_pretrained(a__					)
              self.assertIsNotNone(a__					)
  
  
  
  
  
  
  @require_tf
  class 						lowercase     (   unittest.TestCase):
  
  
  
  
  
  
        @slow
        def    A__(self					)       ->   List[str]:
           SCREAMING_SNAKE_CASE__			:				str       						= TFMPNetModel.from_pretrained('''microsoft/mpnet-base'''					)
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= tf.constant([[0, 1, 2, 3, 4, 5]]					)
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= model(a__					)[0]
  
           SCREAMING_SNAKE_CASE__			:				str       						= [1, 6, 7_6_8]
           self.assertEqual(output.shape ,  a__					)
  
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= tf.constant(
               [
                   [
                       [-0.1_06_71_72, 0.08_21_64_73, 0.0_02_45_43],
                       [-0.03_46_58_79, 0.8_35_41_18, -0.03_25_22_88],
                       [-0.06_56_94_76, -0.12_42_41_11, -0.0_49_44_36],
                   ]
               ]					)
           tf.debugging.assert_near(output[:, :3, :3] ,  a__ ,  atol=1E-4					)
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  import os
  import re
  import unicodedata
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  
  import sentencepiece as spm
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  lowercase__			=	logging.get_logger(__name__)
  lowercase__			=	{'''vocab_file''': '''spiece.model'''}
  
  lowercase__			=	{
      '''vocab_file''': {
          '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
          '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
      },
  }
  
  lowercase__			=	{
      '''google/fnet-base''': 5_1_2,
      '''google/fnet-large''': 5_1_2,
  }
  
  lowercase__			=	'''▁'''
  
  
  
  
  
  
  class 						lowercase     (   UpperCAmelCase_):
        __lowercase 		=  VOCAB_FILES_NAMES
        __lowercase 		=  PRETRAINED_VOCAB_FILES_MAP
        __lowercase 		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        __lowercase 		=  ['''input_ids''', '''token_type_ids''']
  
  
  
  
  
  
        def __init__(self ,  a__ ,  a__=False ,  a__=True ,  a__=True ,  a__="<unk>" ,  a__="[SEP]" ,  a__="<pad>" ,  a__="[CLS]" ,  a__="[MASK]" ,  a__ = None ,  **a__ ,  )       ->   None:
           # Mask token behave like a normal word, i.e. include the space before it and
           # is included in the raw text, there should be a match in a non-normalized sentence.
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= (
               AddedToken(a__ ,  lstrip=a__ ,  rstrip=a__ ,  normalized=a__					)
               if isinstance(a__ ,  a__					)
               else mask_token
           )
  
           SCREAMING_SNAKE_CASE__			:				Any       						= {} if sp_model_kwargs is None else sp_model_kwargs
  
           super().__init__(
               do_lower_case=a__ ,  remove_space=a__ ,  keep_accents=a__ ,  unk_token=a__ ,  sep_token=a__ ,  pad_token=a__ ,  cls_token=a__ ,  mask_token=a__ ,  sp_model_kwargs=self.sp_model_kwargs ,  **a__ ,  )
  
           SCREAMING_SNAKE_CASE__			:				Dict       						= do_lower_case
           SCREAMING_SNAKE_CASE__			:				Tuple       						= remove_space
           SCREAMING_SNAKE_CASE__			:				List[str]       						= keep_accents
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= vocab_file
  
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
           self.sp_model.Load(a__					)
  
  
  
  
  
  
        @property
        def    A__(self					)       ->   Union[str, Any]:
           return len(self.sp_model					)
  
  
  
  
  
  
        def    A__(self					)       ->   Any:
           SCREAMING_SNAKE_CASE__			:				int       						= {self.convert_ids_to_tokens(a__					): i for i in range(self.vocab_size					)}
           vocab.update(self.added_tokens_encoder					)
           return vocab
  
  
  
  
  
  
        def __getstate__(self					)       ->   Any:
           SCREAMING_SNAKE_CASE__			:				List[str]       						= self.__dict__.copy()
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= None
           return state
  
  
  
  
  
  
        def __setstate__(self ,  a__					)       ->   int:
           SCREAMING_SNAKE_CASE__			:				Any       						= d
  
           # for backward compatibility
           if not hasattr(self ,  '''sp_model_kwargs'''					):
              SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= {}
  
           SCREAMING_SNAKE_CASE__			:				int       						= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
           self.sp_model.Load(self.vocab_file					)
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   str:
           if self.remove_space:
              SCREAMING_SNAKE_CASE__			:				List[str]       						= ''' '''.join(inputs.strip().split()					)
           else:
              SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= inputs
           SCREAMING_SNAKE_CASE__			:				Dict       						= outputs.replace('''``''' ,  '''"'''					).replace('''\'\'''' ,  '''"'''					)
  
           if not self.keep_accents:
              SCREAMING_SNAKE_CASE__			:				Dict       						= unicodedata.normalize('''NFKD''' ,  a__					)
              SCREAMING_SNAKE_CASE__			:				List[Any]       						= ''''''.join([c for c in outputs if not unicodedata.combining(a__					)]					)
           if self.do_lower_case:
              SCREAMING_SNAKE_CASE__			:				str       						= outputs.lower()
  
           return outputs
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   List[str]:
           SCREAMING_SNAKE_CASE__			:				Dict       						= self.preprocess_text(a__					)
           SCREAMING_SNAKE_CASE__			:				List[str]       						= self.sp_model.encode(a__ ,  out_type=a__					)
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= []
           for piece in pieces:
              if len(a__					) > 1 and piece[-1] == str(''','''					) and piece[-2].isdigit():
                 SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= self.sp_model.EncodeAsPieces(piece[:-1].replace(a__ ,  ''''''					)					)
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                    if len(cur_pieces[0]					) == 1:
                       SCREAMING_SNAKE_CASE__			:				List[str]       						= cur_pieces[1:]
                    else:
                       SCREAMING_SNAKE_CASE__			:				str       						= cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]					)
                 new_pieces.extend(a__					)
              else:
                 new_pieces.append(a__					)
  
           return new_pieces
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   str:
           return self.sp_model.PieceToId(a__					)
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   Optional[int]:
           return self.sp_model.IdToPiece(a__					)
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   Optional[Any]:
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= []
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= ''''''
           SCREAMING_SNAKE_CASE__			:				Any       						= False
           for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                 if not prev_is_special:
                    out_string += " "
                 out_string += self.sp_model.decode(a__					) + token
                 SCREAMING_SNAKE_CASE__			:				List[Any]       						= True
                 SCREAMING_SNAKE_CASE__			:				int       						= []
              else:
                 current_sub_tokens.append(a__					)
                 SCREAMING_SNAKE_CASE__			:				Dict       						= False
           out_string += self.sp_model.decode(a__					)
           return out_string.strip()
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = False ,  a__ = None ,  a__ = True ,  **a__ ,  )       ->   str:
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= kwargs.pop('''use_source_tokenizer''' ,  a__					)
  
           SCREAMING_SNAKE_CASE__			:				int       						= self.convert_ids_to_tokens(a__ ,  skip_special_tokens=a__					)
  
           # To avoid mixing byte-level and unicode for byte-level BPT
           # we need to build string separately for added tokens and byte-level tokens
           # cf. https://github.com/huggingface/transformers/issues/1133
           SCREAMING_SNAKE_CASE__			:				Dict       						= []
           SCREAMING_SNAKE_CASE__			:				List[str]       						= []
           for token in filtered_tokens:
              if skip_special_tokens and token in self.all_special_ids:
                 continue
              if token in self.added_tokens_encoder:
                 if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(a__					)					)
                    SCREAMING_SNAKE_CASE__			:				List[str]       						= []
                 sub_texts.append(a__					)
              else:
                 current_sub_text.append(a__					)
           if current_sub_text:
              sub_texts.append(self.convert_tokens_to_string(a__					)					)
  
           # Mimic the behavior of the Rust tokenizer:
           # No space after <unk>
           if spaces_between_special_tokens:
              SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= re.sub(R'''(<unk>) ''' ,  R'''\1''' ,  ''' '''.join(a__					)					)
           else:
              SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= ''''''.join(a__					)
  
           SCREAMING_SNAKE_CASE__			:				int       						= (
               clean_up_tokenization_spaces
               if clean_up_tokenization_spaces is not None
               else self.clean_up_tokenization_spaces
           )
           if clean_up_tokenization_spaces:
              SCREAMING_SNAKE_CASE__			:				str       						= self.clean_up_tokenization(a__					)
              return clean_text
           else:
              return text
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
           SCREAMING_SNAKE_CASE__			:				str       						= [self.sep_token_id]
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= [self.cls_token_id]
           if token_ids_a is None:
              return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None ,  a__ = False					)       ->   List[int]:
  
           if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=a__ ,  token_ids_a=a__ ,  already_has_special_tokens=a__					)
  
           if token_ids_a is not None:
              return [1] + ([0] * len(a__					)) + [1] + ([0] * len(a__					)) + [1]
           return [1] + ([0] * len(a__					)) + [1]
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
           SCREAMING_SNAKE_CASE__			:				List[str]       						= [self.sep_token_id]
           SCREAMING_SNAKE_CASE__			:				Tuple       						= [self.cls_token_id]
  
           if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
           return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   Tuple[str]:
           if not os.path.isdir(a__					):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
              return
           SCREAMING_SNAKE_CASE__			:				str       						= os.path.join(
               a__ ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
  
           if os.path.abspath(self.vocab_file					) != os.path.abspath(a__					) and os.path.isfile(self.vocab_file					):
              copyfile(self.vocab_file ,  a__					)
           elif not os.path.isfile(self.vocab_file					):
              with open(a__ ,  '''wb'''					) as fi:
                 SCREAMING_SNAKE_CASE__			:				int       						= self.sp_model.serialized_model_proto()
                 fi.write(a__					)
  
           return (out_vocab_file,)
  
  