



"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


lowerCamelCase				:     Any								=  logging.get_logger(__name__)

lowerCamelCase				:     Tuple								=  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase				:     Tuple								=  {
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """allenai/longformer-base-4096""": (
            """https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase				:     Optional[int]								=  {
    """allenai/longformer-base-4096""": 4_0_9_6,
    """allenai/longformer-large-4096""": 4_0_9_6,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4_0_9_6,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4_0_9_6,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4_0_9_6,
}




class    __snake_case(					__A       ):
    _A       = VOCAB_FILES_NAMES
    _A       = PRETRAINED_VOCAB_FILES_MAP
    _A       = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _A       = ['''input_ids''', '''attention_mask''']
    _A       = LongformerTokenizer




    def __init__(   self    ,							A_=None    ,							A_=None    ,							A_=None    ,							A_="replace"    ,							A_="<s>"    ,							A_="</s>"    ,							A_="</s>"    ,							A_="<s>"    ,							A_="<unk>"    ,							A_="<pad>"    ,							A_="<mask>"    ,							A_=False    ,							A_=True    ,							**A_    ,							):





      '''simple docstring'''



      super().__init__(
          a__    ,							a__    ,							tokenizer_file=a__    ,							errors=a__    ,							bos_token=a__    ,							eos_token=a__    ,							sep_token=a__    ,							cls_token=a__    ,							unk_token=a__    ,							pad_token=a__    ,							mask_token=a__    ,							add_prefix_space=a__    ,							trim_offsets=a__    ,							**a__    ,							)

      _SCREAMING_SNAKE_CASE        =			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
      if pre_tok_state.get('''add_prefix_space'''    ,							a__			) != add_prefix_space:
        _SCREAMING_SNAKE_CASE        =			getattr(a__    ,							pre_tok_state.pop('''type'''			)			)
        _SCREAMING_SNAKE_CASE        =			add_prefix_space
        _SCREAMING_SNAKE_CASE        =			pre_tok_class(**a__			)

      _SCREAMING_SNAKE_CASE        =			add_prefix_space

      _SCREAMING_SNAKE_CASE        =			'''post_processor'''
      _SCREAMING_SNAKE_CASE        =			getattr(self.backend_tokenizer    ,							a__    ,							a__			)
      if tokenizer_component_instance:
        _SCREAMING_SNAKE_CASE        =			json.loads(tokenizer_component_instance.__getstate__()			)

        # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
        if "sep" in state:
          _SCREAMING_SNAKE_CASE        =			tuple(state['''sep''']			)
        if "cls" in state:
          _SCREAMING_SNAKE_CASE        =			tuple(state['''cls''']			)

        _SCREAMING_SNAKE_CASE        =			False

        if state.get('''add_prefix_space'''    ,							a__			) != add_prefix_space:
          _SCREAMING_SNAKE_CASE        =			add_prefix_space
          _SCREAMING_SNAKE_CASE        =			True

        if state.get('''trim_offsets'''    ,							a__			) != trim_offsets:
          _SCREAMING_SNAKE_CASE        =			trim_offsets
          _SCREAMING_SNAKE_CASE        =			True

        if changes_to_apply:
          _SCREAMING_SNAKE_CASE        =			getattr(a__    ,							state.pop('''type'''			)			)
          _SCREAMING_SNAKE_CASE        =			component_class(**a__			)
          setattr(self.backend_tokenizer    ,							a__    ,							a__			)




    @property
    def   A   (   self			):





      '''simple docstring'''



      if self._mask_token is None:
        if self.verbose:
          logger.error('''Using mask_token, but it is not set yet.'''			)
        return None
      return str(self._mask_token			)




    @mask_token.setter
    def   A   (   self    ,							A_			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			AddedToken(a__    ,							lstrip=a__    ,							rstrip=a__			) if isinstance(a__    ,							a__			) else value
      _SCREAMING_SNAKE_CASE        =			value




    def   A   (   self    ,							*A_    ,							**A_			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			kwargs.get('''is_split_into_words'''    ,							a__			)
      assert self.add_prefix_space or not is_split_into_words, (
          F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*a__    ,							**a__			)




    def   A   (   self    ,							*A_    ,							**A_			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			kwargs.get('''is_split_into_words'''    ,							a__			)

      assert self.add_prefix_space or not is_split_into_words, (
          F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*a__    ,							**a__			)




    def   A   (   self    ,							A_    ,							A_ = None			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			self._tokenizer.model.save(a__    ,							name=a__			)
      return tuple(a__			)




    def   A   (   self    ,							A_    ,							A_=None			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
      if token_ids_a is None:
        return output

      return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




    def   A   (   self    ,							A_    ,							A_ = None			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			[self.sep_token_id]
      _SCREAMING_SNAKE_CASE        =			[self.cls_token_id]

      if token_ids_a is None:
        return len(cls + token_ids_a + sep			) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]







"""simple docstring"""






from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_      =				logging.get_logger(__name__)

lowerCAmelCase_      =				{
    '''microsoft/prophetnet-large-uncased''': (
        '''https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json'''
    ),
}



class 						lowerCAmelCase       (   snake_case				):
       lowerCAmelCase__    			=	"""prophetnet"""
       lowerCAmelCase__    			=	["""past_key_values"""]
       lowerCAmelCase__    			=	{
           """num_attention_heads""": """num_encoder_attention_heads""",
       }






       def __init__( self     ,	a__ = 0.1     ,	a__ = "gelu"     ,	a__ = 3_05_22     ,	a__ = 10_24     ,	a__ = 40_96     ,	a__ = 12     ,	a__ = 16     ,	a__ = 40_96     ,	a__ = 12     ,	a__ = 16     ,	a__ = 0.1     ,	a__ = 0.1     ,	a__ = 5_12     ,	a__ = 0.02     ,	a__ = True     ,	a__ = True     ,	a__ = 0     ,	a__ = 2     ,	a__ = 32     ,	a__ = 1_28     ,	a__ = False     ,	a__ = 0.0     ,	a__ = True     ,	a__ = 0     ,	a__ = 1     ,	a__ = 2     ,	**a__     ,	):
              _UpperCAmelCase			     =  vocab_size
              _UpperCAmelCase			     =  hidden_size
              _UpperCAmelCase			     =  encoder_ffn_dim
              _UpperCAmelCase			     =  num_encoder_layers
              _UpperCAmelCase			     =  num_encoder_attention_heads
              _UpperCAmelCase			     =  decoder_ffn_dim
              _UpperCAmelCase			     =  num_decoder_layers
              _UpperCAmelCase			     =  num_decoder_attention_heads
              _UpperCAmelCase			     =  max_position_embeddings
              _UpperCAmelCase			     =  init_std  # Normal(0, this parameter)
              _UpperCAmelCase			     =  activation_function

              # parameters for prophetnet
              _UpperCAmelCase			     =  ngram
              _UpperCAmelCase			     =  num_buckets
              _UpperCAmelCase			     =  relative_max_distance
              _UpperCAmelCase			     =  disable_ngram_loss
              _UpperCAmelCase			     =  eps

              # 3 Types of Dropout
              _UpperCAmelCase			     =  attention_dropout
              _UpperCAmelCase			     =  activation_dropout
              _UpperCAmelCase			     =  dropout

              _UpperCAmelCase			     =  use_cache

              super().__init__(
                  pad_token_id=a__     ,	bos_token_id=a__     ,	eos_token_id=a__     ,	is_encoder_decoder=a__     ,	add_cross_attention=a__     ,	decoder_start_token_id=a__     ,	**a__     ,	)






       @property
       def 				__A			( self  ):
              return self.num_encoder_layers + self.num_decoder_layers






       @num_hidden_layers.setter
       def 				__A			( self     ,	a__  ):
              raise NotImplementedError(
                  'This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'
                  ' `num_decoder_layers`.'  )






