





'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


__lowerCAmelCase							:	List[Any]			=logging.get_logger(__name__)


__lowerCAmelCase							:	List[Any]			={"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

# See all MVP models at https://huggingface.co/models?filter=mvp
__lowerCAmelCase							:	int			={
    "vocab_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json",
    },
    "added_tokens.json": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json",
    },
    "merges_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json",
    },
}

__lowerCAmelCase							:	str			={
    "RUCAIBox/mvp": 1024,
}






class    UpperCAmelCase	(       UpperCamelCase__				):
 __lowercase   		=				VOCAB_FILES_NAMES
 __lowercase   		=				PRETRAINED_VOCAB_FILES_MAP
 __lowercase   		=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase   		=				["""input_ids""", """attention_mask"""]
 __lowercase   		=				MvpTokenizer
 def __init__(				self      :Optional[Any]						,       lowercase_      :List[str]=None						,       lowercase_      :Any=None						,       lowercase_      :int=None						,       lowercase_      :Dict="replace"						,       lowercase_      :List[Any]="<s>"						,       lowercase_      :int="</s>"						,       lowercase_      :Optional[int]="</s>"						,       lowercase_      :Tuple="<s>"						,       lowercase_      :Optional[int]="<unk>"						,       lowercase_      :Union[str, Any]="<pad>"						,       lowercase_      :str="<mask>"						,       lowercase_      :List[Any]=False						,       lowercase_      :str=True						,       **lowercase_      :str						,       )->							int:
     super().__init__(
         lowercase_						,       lowercase_						,       tokenizer_file=lowercase_						,       errors=lowercase_						,       bos_token=lowercase_						,       eos_token=lowercase_						,       sep_token=lowercase_						,       cls_token=lowercase_						,       unk_token=lowercase_						,       pad_token=lowercase_						,       mask_token=lowercase_						,       add_prefix_space=lowercase_						,       trim_offsets=lowercase_						,       **lowercase_						,       )

     A__				=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
     if pre_tok_state.get("add_prefix_space"						,       lowercase_							) != add_prefix_space:
         A__				=			getattr(lowercase_						,       pre_tok_state.pop("type"							)							)
         A__				=			add_prefix_space
         A__				=			pre_tok_class(**lowercase_							)

     A__				=			add_prefix_space

     # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
     A__				=			"post_processor"
     A__				=			getattr(self.backend_tokenizer						,       lowercase_						,       lowercase_							)
     if tokenizer_component_instance:
         A__				=			json.loads(tokenizer_component_instance.__getstate__()							)

         # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
         if "sep" in state:
             A__				=			tuple(state["sep"]							)
         if "cls" in state:
             A__				=			tuple(state["cls"]							)

         A__				=			False

         if state.get("add_prefix_space"						,       lowercase_							) != add_prefix_space:
             A__				=			add_prefix_space
             A__				=			True

         if state.get("trim_offsets"						,       lowercase_							) != trim_offsets:
             A__				=			trim_offsets
             A__				=			True

         if changes_to_apply:
             A__				=			getattr(lowercase_						,       state.pop("type"							)							)
             A__				=			component_class(**lowercase_							)
             setattr(self.backend_tokenizer						,       lowercase_						,       lowercase_							)
 @property
 def      UpperCAmelCase_							(				self      :Tuple							)->							str:
     if self._mask_token is None:
         if self.verbose:
             logger.error("Using mask_token, but it is not set yet."							)
         return None
     return str(self._mask_token							)
 @mask_token.setter
 def      UpperCAmelCase_							(				self      :Tuple						,       lowercase_      :Optional[Any]							)->							Any:
     A__				=			AddedToken(lowercase_						,       lstrip=lowercase_						,       rstrip=lowercase_							) if isinstance(lowercase_						,       lowercase_							) else value
     A__				=			value
 def      UpperCAmelCase_							(				self      :Dict						,       *lowercase_      :Any						,       **lowercase_      :List[Any]							)->							BatchEncoding:
     A__				=			kwargs.get("is_split_into_words"						,       lowercase_							)

     if is_split_into_words and not self.add_prefix_space:
         raise ValueError(
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."							)

     return super()._batch_encode_plus(*lowercase_						,       **lowercase_							)
 def      UpperCAmelCase_							(				self      :List[Any]						,       *lowercase_      :Dict						,       **lowercase_      :int							)->							BatchEncoding:
     A__				=			kwargs.get("is_split_into_words"						,       lowercase_							)

     if is_split_into_words and not self.add_prefix_space:
         raise ValueError(
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."							)

     return super()._encode_plus(*lowercase_						,       **lowercase_							)
 def      UpperCAmelCase_							(				self      :List[str]						,       lowercase_      :str						,       lowercase_      :Optional[str] = None							)->							Tuple[str]:
     A__				=			self._tokenizer.model.save(lowercase_						,       name=lowercase_							)
     return tuple(lowercase_							)
 def      UpperCAmelCase_							(				self      :Optional[Any]						,       lowercase_      :List[Any]						,       lowercase_      :Union[str, Any]=None							)->							Optional[Any]:
     A__				=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
     if token_ids_a is None:
         return output

     return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



 def      UpperCAmelCase_							(				self      :str						,       lowercase_      :List[int]						,       lowercase_      :Optional[List[int]] = None							)->							List[int]:
     A__				=			[self.sep_token_id]
     A__				=			[self.cls_token_id]

     if token_ids_a is None:
         return len(cls + token_ids_a + sep							) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]










'''simple docstring'''

import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
  from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
  from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
  from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


__lowerCAmelCase							:	Dict			=DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

__lowerCAmelCase							:	List[Any]			="main"
# Default branch name
__lowerCAmelCase							:	int			="f2c752cfc5c0ab6f4bdec59acea69eefbee381c2"
# One particular commit (not the top of `main`)
__lowerCAmelCase							:	List[Any]			="aaaaaaa"
# This commit does not exist, so we should 404.

__lowerCAmelCase							:	Optional[int]			="d9e9f15bc825e4b2c9249e9578f884bbcb5e3684"
# Sha-1 of config.json on the top of `main`, for checking purposes
__lowerCAmelCase							:	Dict			="4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3"





@contextlib.contextmanager
def 						UpperCamelCase				(							):
    print("Welcome!"       )
    yield
    print("Bye!"       )





@contextlib.contextmanager
def 						UpperCamelCase				(							):
    print("Bonjour!"       )
    yield
    print("Au revoir!"       )






class    UpperCAmelCase	(       unittest.TestCase				):
 def      UpperCAmelCase_							(				self      :str							)->							List[str]:
     # If the spec is missing, importlib would not be able to import the module dynamically.
     assert transformers.__spec__ is not None
     assert importlib.util.find_spec("transformers"							) is not None






class    UpperCAmelCase	(       unittest.TestCase				):
 @unittest.mock.patch("sys.stdout"						,       new_callable=io.StringIO							)
 def      UpperCAmelCase_							(				self      :Union[str, Any]						,       lowercase_      :Union[str, Any]							)->							Any:
     with ContextManagers([]							):
         print("Transformers are awesome!"							)
     # The print statement adds a new line at the end of the output
     self.assertEqual(mock_stdout.getvalue()						,       "Transformers are awesome!\n"							)
 @unittest.mock.patch("sys.stdout"						,       new_callable=io.StringIO							)
 def      UpperCAmelCase_							(				self      :Dict						,       lowercase_      :Optional[Any]							)->							Tuple:
     with ContextManagers([context_en()]							):
         print("Transformers are awesome!"							)
     # The output should be wrapped with an English welcome and goodbye
     self.assertEqual(mock_stdout.getvalue()						,       "Welcome!\nTransformers are awesome!\nBye!\n"							)
 @unittest.mock.patch("sys.stdout"						,       new_callable=io.StringIO							)
 def      UpperCAmelCase_							(				self      :Union[str, Any]						,       lowercase_      :int							)->							Union[str, Any]:
     with ContextManagers([context_fr(), context_en()]							):
         print("Transformers are awesome!"							)
     # The output should be wrapped with an English and French welcome and goodbye
     self.assertEqual(mock_stdout.getvalue()						,       "Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n"							)
 @require_torch
 def      UpperCAmelCase_							(				self      :int							)->							Dict:
     self.assertEqual(find_labels(lowercase_							)						,       ["labels"]							)
     self.assertEqual(find_labels(lowercase_							)						,       ["labels", "next_sentence_label"]							)
     self.assertEqual(find_labels(lowercase_							)						,       ["start_positions", "end_positions"]							)






     class    UpperCAmelCase	(       UpperCamelCase__				):
      pass

     self.assertEqual(find_labels(lowercase_							)						,       ["labels"]							)
 @require_tf
 def      UpperCAmelCase_							(				self      :Union[str, Any]							)->							Union[str, Any]:
     self.assertEqual(find_labels(lowercase_							)						,       ["labels"]							)
     self.assertEqual(find_labels(lowercase_							)						,       ["labels", "next_sentence_label"]							)
     self.assertEqual(find_labels(lowercase_							)						,       ["start_positions", "end_positions"]							)






     class    UpperCAmelCase	(       UpperCamelCase__				):
      pass

     self.assertEqual(find_labels(lowercase_							)						,       ["labels"]							)



 @require_flax
 def      UpperCAmelCase_							(				self      :Dict							)->							str:
     # Flax models don't have labels
     self.assertEqual(find_labels(lowercase_							)						,       []							)
     self.assertEqual(find_labels(lowercase_							)						,       []							)
     self.assertEqual(find_labels(lowercase_							)						,       []							)






     class    UpperCAmelCase	(       UpperCamelCase__				):
      pass

     self.assertEqual(find_labels(lowercase_							)						,       []							)



