
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


_A							=       logging.get_logger(__name__)

_A							=       {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_A							=       {
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

_A							=       {
    '''microsoft/mpnet-base''': 512,
}

_A							=       {
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}

class 		lowercase_							(							__SCREAMING_SNAKE_CASE					):
 A__    :					Tuple    =     VOCAB_FILES_NAMES
 A__    :					List[str]    =     PRETRAINED_VOCAB_FILES_MAP
 A__    :					List[str]    =     PRETRAINED_INIT_CONFIGURATION
 A__    :					Optional[int]    =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 A__    :					Optional[Any]    =     MPNetTokenizer
 A__    :					Optional[Any]    =     ["""input_ids""", """attention_mask"""]

 def __init__(   self  ,						__UpperCamelCase=None  ,						__UpperCamelCase=None  ,						__UpperCamelCase=True  ,						__UpperCamelCase="<s>"  ,						__UpperCamelCase="</s>"  ,						__UpperCamelCase="</s>"  ,						__UpperCamelCase="<s>"  ,						__UpperCamelCase="[UNK]"  ,						__UpperCamelCase="<pad>"  ,						__UpperCamelCase="<mask>"  ,						__UpperCamelCase=True  ,						__UpperCamelCase=None  ,						**__UpperCamelCase  ,						):




        """simple docstring"""


        super().__init__(
            __UpperCamelCase  ,						tokenizer_file=__UpperCamelCase  ,						do_lower_case=__UpperCamelCase  ,						bos_token=__UpperCamelCase  ,						eos_token=__UpperCamelCase  ,						sep_token=__UpperCamelCase  ,						cls_token=__UpperCamelCase  ,						unk_token=__UpperCamelCase  ,						pad_token=__UpperCamelCase  ,						mask_token=__UpperCamelCase  ,						tokenize_chinese_chars=__UpperCamelCase  ,						strip_accents=__UpperCamelCase  ,						**__UpperCamelCase  ,						)

        UpperCamelCase_											=							json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
        if (
            pre_tok_state.get("""lowercase"""  ,						__UpperCamelCase  ) != do_lower_case
            or pre_tok_state.get("""strip_accents"""  ,						__UpperCamelCase  ) != strip_accents
        ):
               UpperCamelCase_											=							getattr(__UpperCamelCase  ,						pre_tok_state.pop("""type"""  )  )
               UpperCamelCase_											=							do_lower_case
               UpperCamelCase_											=							strip_accents
               UpperCamelCase_											=							pre_tok_class(**__UpperCamelCase  )

        UpperCamelCase_											=							do_lower_case

 @property
 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        if self._mask_token is None:
               if self.verbose:
                      logger.error("""Using mask_token, but it is not set yet."""  )
               return None
        return str(self._mask_token  )

 @mask_token.setter
 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							AddedToken(__UpperCamelCase  ,						lstrip=__UpperCamelCase  ,						rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) else value
        UpperCamelCase_											=							value

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase=None  ):




        """simple docstring"""


        UpperCamelCase_											=							[self.bos_token_id] + token_ids_a + [self.eos_token_id]
        if token_ids_a is None:
               return output

        return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase = None  ):




        """simple docstring"""


        UpperCamelCase_											=							[self.sep_token_id]
        UpperCamelCase_											=							[self.cls_token_id]

        if token_ids_a is None:
               return len(cls + token_ids_a + sep  ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase = None  ):




        """simple docstring"""


        UpperCamelCase_											=							self._tokenizer.model.save(__UpperCamelCase  ,						name=__UpperCamelCase  )
        return tuple(__UpperCamelCase  )





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_A							=       {'''configuration_dinat''': ['''DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''DinatConfig''']}


try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _A							=       [
          '''DINAT_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''DinatForImageClassification''',
          '''DinatModel''',
          '''DinatPreTrainedModel''',
          '''DinatBackbone''',
      ]

if TYPE_CHECKING:
      from .configuration_dinat import DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP, DinatConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_dinat import (
                DINAT_PRETRAINED_MODEL_ARCHIVE_LIST,
                DinatBackbone,
                DinatForImageClassification,
                DinatModel,
                DinatPreTrainedModel,
            )

else:
      import sys

      _A							=       _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



