


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__lowerCAmelCase		       =							logging.get_logger(__name__)

__lowerCAmelCase		       =							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCAmelCase		       =							[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__lowerCAmelCase		       =							{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__lowerCAmelCase		       =							{F'''funnel-transformer/{name}''': 5_12 for name in _model_names}
__lowerCAmelCase		       =							{F'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}





class   __a			(    __UpperCamelCase      ):
 __lowercase :							Optional[Any]        = VOCAB_FILES_NAMES
 __lowercase :							Optional[int]        = PRETRAINED_VOCAB_FILES_MAP
 __lowercase :							Any        = PRETRAINED_INIT_CONFIGURATION
 __lowercase :							Optional[Any]        = FunnelTokenizer
 __lowercase :							Optional[int]        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase :							int        = 2
 def __init__(   self				, lowerCAmelCase__=None				, lowerCAmelCase__=None				, lowerCAmelCase__=True				, lowerCAmelCase__="<unk>"				, lowerCAmelCase__="<sep>"				, lowerCAmelCase__="<pad>"				, lowerCAmelCase__="<cls>"				, lowerCAmelCase__="<mask>"				, lowerCAmelCase__="<s>"				, lowerCAmelCase__="</s>"				, lowerCAmelCase__=True				, lowerCAmelCase__=True				, lowerCAmelCase__=None				, lowerCAmelCase__="##"				, **lowerCAmelCase__				, )					->				List[Any]:




        '''simple docstring'''
        super().__init__(
            lowerCAmelCase__				, tokenizer_file=lowerCAmelCase__				, do_lower_case=lowerCAmelCase__				, unk_token=lowerCAmelCase__				, sep_token=lowerCAmelCase__				, pad_token=lowerCAmelCase__				, cls_token=lowerCAmelCase__				, mask_token=lowerCAmelCase__				, bos_token=lowerCAmelCase__				, eos_token=lowerCAmelCase__				, clean_text=lowerCAmelCase__				, tokenize_chinese_chars=lowerCAmelCase__				, strip_accents=lowerCAmelCase__				, wordpieces_prefix=lowerCAmelCase__				, **lowerCAmelCase__				, )

        lowercase__:   Tuple  			=   json.loads(self.backend_tokenizer.normalizer.__getstate__() )
        if (
            normalizer_state.get('lowercase'				, lowerCAmelCase__ ) != do_lower_case
            or normalizer_state.get('strip_accents'				, lowerCAmelCase__ ) != strip_accents
            or normalizer_state.get('handle_chinese_chars'				, lowerCAmelCase__ ) != tokenize_chinese_chars
        ):
               lowercase__:   Any  			=   getattr(lowerCAmelCase__				, normalizer_state.pop('type' ) )
               lowercase__:   Optional[int]  			=   do_lower_case
               lowercase__:   Union[str, Any]  			=   strip_accents
               lowercase__:   Optional[int]  			=   tokenize_chinese_chars
               lowercase__:   str  			=   normalizer_class(**lowerCAmelCase__ )

        lowercase__:   List[str]  			=   do_lower_case
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__=None )					->				int:




        '''simple docstring'''
        lowercase__:   List[Any]  			=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

        return output
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None )					->				List[int]:




        '''simple docstring'''
        lowercase__:   Tuple  			=   [self.sep_token_id]
        lowercase__:   Tuple  			=   [self.cls_token_id]
        if token_ids_a is None:
               return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0]
        return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None )					->				Tuple[str]:




        '''simple docstring'''
        lowercase__:   Any  			=   self._tokenizer.model.save(lowerCAmelCase__				, name=lowerCAmelCase__ )
        return tuple(lowerCAmelCase__ )






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__lowerCAmelCase		       =							{
    '''configuration_canine''': ['''CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''CanineConfig'''],
    '''tokenization_canine''': ['''CanineTokenizer'''],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowerCAmelCase		       =							[
     '''CANINE_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''CanineForMultipleChoice''',
     '''CanineForQuestionAnswering''',
     '''CanineForSequenceClassification''',
     '''CanineForTokenClassification''',
     '''CanineLayer''',
     '''CanineModel''',
     '''CaninePreTrainedModel''',
     '''load_tf_weights_in_canine''',
 ]


if TYPE_CHECKING:
 from .configuration_canine import CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP, CanineConfig
 from .tokenization_canine import CanineTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_canine import (
      CANINE_PRETRAINED_MODEL_ARCHIVE_LIST,
      CanineForMultipleChoice,
      CanineForQuestionAnswering,
      CanineForSequenceClassification,
      CanineForTokenClassification,
      CanineLayer,
      CanineModel,
      CaninePreTrainedModel,
      load_tf_weights_in_canine,
  )


else:
 import sys

 __lowerCAmelCase		       =							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


