




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


snake_case_ : Optional[int]    		=       logging.get_logger(__name__)

snake_case_ : str    		=       {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

snake_case_ : int    		=       [
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

snake_case_ : Dict    		=       {
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
snake_case_ : int    		=       {f"funnel-transformer/{name}": 512 for name in _model_names}
snake_case_ : Tuple    		=       {f"funnel-transformer/{name}": {"do_lower_case": True} for name in _model_names}







class __snake_case			(			a		):
   UpperCAmelCase__  :    Optional[Any]          =  VOCAB_FILES_NAMES
   UpperCAmelCase__  :    Dict          =  PRETRAINED_VOCAB_FILES_MAP
   UpperCAmelCase__  :    Dict          =  PRETRAINED_INIT_CONFIGURATION
   UpperCAmelCase__  :    Optional[int]          =  FunnelTokenizer
   UpperCAmelCase__  :    List[Any]          =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCAmelCase__  :    int          =  2







   def __init__(      self		:						List[str]					,						_snake_case		:						str=None					,						_snake_case		:						Tuple=None					,						_snake_case		:						int=True					,						_snake_case		:						Optional[Any]="<unk>"					,						_snake_case		:						List[Any]="<sep>"					,						_snake_case		:						Dict="<pad>"					,						_snake_case		:						List[Any]="<cls>"					,						_snake_case		:						str="<mask>"					,						_snake_case		:						Optional[Any]="<s>"					,						_snake_case		:						List[str]="</s>"					,						_snake_case		:						int=True					,						_snake_case		:						Any=True					,						_snake_case		:						int=None					,						_snake_case		:						List[Any]="##"					,						**_snake_case		:						Tuple					,						):

        """simple docstring"""


        super().__init__(
            _snake_case					,						tokenizer_file=_snake_case					,						do_lower_case=_snake_case					,						unk_token=_snake_case					,						sep_token=_snake_case					,						pad_token=_snake_case					,						cls_token=_snake_case					,						mask_token=_snake_case					,						bos_token=_snake_case					,						eos_token=_snake_case					,						clean_text=_snake_case					,						tokenize_chinese_chars=_snake_case					,						strip_accents=_snake_case					,						wordpieces_prefix=_snake_case					,						**_snake_case					,						)

        UpperCAmelCase_          =							json.loads(self.backend_tokenizer.normalizer.__getstate__())
        if (
            normalizer_state.get('''lowercase'''					,						_snake_case) != do_lower_case
            or normalizer_state.get('''strip_accents'''					,						_snake_case) != strip_accents
            or normalizer_state.get('''handle_chinese_chars'''					,						_snake_case) != tokenize_chinese_chars
        ):
             UpperCAmelCase_          =							getattr(_snake_case					,						normalizer_state.pop('''type'''))
             UpperCAmelCase_          =							do_lower_case
             UpperCAmelCase_          =							strip_accents
             UpperCAmelCase_          =							tokenize_chinese_chars
             UpperCAmelCase_          =							normalizer_class(**_snake_case)

        UpperCAmelCase_          =							do_lower_case







   def lowerCamelCase						(      self		:						Tuple					,						_snake_case		:						str					,						_snake_case		:						Optional[Any]=None):

        """simple docstring"""


        UpperCAmelCase_          =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

        return output







   def lowerCamelCase						(      self		:						List[Any]					,						_snake_case		:						List[int]					,						_snake_case		:						Optional[List[int]] = None):

        """simple docstring"""


        UpperCAmelCase_          =							[self.sep_token_id]
        UpperCAmelCase_          =							[self.cls_token_id]
        if token_ids_a is None:
             return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
        return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]







   def lowerCamelCase						(      self		:						Tuple					,						_snake_case		:						str					,						_snake_case		:						Optional[str] = None):

        """simple docstring"""


        UpperCAmelCase_          =							self._tokenizer.model.save(_snake_case					,						name=_snake_case)
        return tuple(_snake_case)












import warnings

from ...utils import logging
from .image_processing_glpn import GLPNImageProcessor


lowerCamelCase				:		Union[str, Any]     =   logging.get_logger(__name__)







class   A(						UpperCamelCase   ):



 '''simple docstring'''


 def __init__(				self					:  int						,							*A_					:  str						,							**A_					:  Optional[int]  )   ->	None:

       """simple docstring"""






       warnings.warn(
           'The class GLPNFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'
           ' use GLPNImageProcessor instead.'						,							A_						,							)
       super().__init__(*A_						,							**A_  )



