



'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


UpperCamelCase__:   Tuple								=			logging.get_logger(__name__)

UpperCamelCase__:   Tuple								=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__:   List[Any]								=			[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

UpperCamelCase__:   Optional[Any]								=			{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
UpperCamelCase__:   Dict								=			{F"funnel-transformer/{name}": 512 for name in _model_names}
UpperCamelCase__:   Optional[Any]								=			{F"funnel-transformer/{name}": {"do_lower_case": True} for name in _model_names}

class 				SCREAMING_SNAKE_CASE(							snake_case_			):







	"""simple docstring"""






	lowerCamelCase__			 =       VOCAB_FILES_NAMES
	lowerCamelCase__			 =       PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase__			 =       PRETRAINED_INIT_CONFIGURATION
	lowerCamelCase__			 =       FunnelTokenizer
	lowerCamelCase__			 =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCamelCase__			 =       2







	def __init__(						self			:     Union[str, Any]		,			__snake_case			:     Optional[Any]=None		,			__snake_case			:     Union[str, Any]=None		,			__snake_case			:     Optional[int]=True		,			__snake_case			:     Tuple="<unk>"		,			__snake_case			:     Optional[int]="<sep>"		,			__snake_case			:     Optional[Any]="<pad>"		,			__snake_case			:     Optional[Any]="<cls>"		,			__snake_case			:     Any="<mask>"		,			__snake_case			:     Union[str, Any]="<s>"		,			__snake_case			:     Tuple="</s>"		,			__snake_case			:     Tuple=True		,			__snake_case			:     str=True		,			__snake_case			:     List[str]=None		,			__snake_case			:     int="##"		,			**__snake_case			:     Any		,			)			->		int:
								super().__init__(
								    __snake_case		,			tokenizer_file=__snake_case		,			do_lower_case=__snake_case		,			unk_token=__snake_case		,			sep_token=__snake_case		,			pad_token=__snake_case		,			cls_token=__snake_case		,			mask_token=__snake_case		,			bos_token=__snake_case		,			eos_token=__snake_case		,			clean_text=__snake_case		,			tokenize_chinese_chars=__snake_case		,			strip_accents=__snake_case		,			wordpieces_prefix=__snake_case		,			**__snake_case		,			)

								UpperCAmelCase :       List[Any] 							=		json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
								if (
								    normalizer_state.get('''lowercase'''		,			__snake_case		) != do_lower_case
								    or normalizer_state.get('''strip_accents'''		,			__snake_case		) != strip_accents
								    or normalizer_state.get('''handle_chinese_chars'''		,			__snake_case		) != tokenize_chinese_chars
								):
															UpperCAmelCase :       Dict 							=		getattr(__snake_case		,			normalizer_state.pop('''type'''		)		)
															UpperCAmelCase :       int 							=		do_lower_case
															UpperCAmelCase :       Union[str, Any] 							=		strip_accents
															UpperCAmelCase :       Union[str, Any] 							=		tokenize_chinese_chars
															UpperCAmelCase :       Tuple 							=		normalizer_class(**__snake_case		)

								UpperCAmelCase :       List[Any] 							=		do_lower_case







	def      A	(						self			:     Any		,			__snake_case			:     Optional[int]		,			__snake_case			:     Optional[Any]=None		)			->		List[Any]:
								UpperCAmelCase :       Dict 							=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

								return output







	def      A	(						self			:     Dict		,			__snake_case			:     str		,			__snake_case			:     Tuple = None		)			->		List[int]:
								UpperCAmelCase :       Optional[int] 							=		[self.sep_token_id]
								UpperCAmelCase :       Tuple 							=		[self.cls_token_id]
								if token_ids_a is None:
															return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
								return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







	def      A	(						self			:     List[Any]		,			__snake_case			:     Dict		,			__snake_case			:     Optional[int] = None		)			->		Tuple[str]:
								UpperCAmelCase :       Union[str, Any] 							=		self._tokenizer.model.save(__snake_case		,			name=__snake_case		)
								return tuple(__snake_case		)













from typing import TYPE_CHECKING

from ...utils import _LazyModule


lowercase  							=					{"processing_wav2vec2_with_lm": ["Wav2Vec2ProcessorWithLM"]}


if TYPE_CHECKING:
	from .processing_wavaveca_with_lm import WavaVecaProcessorWithLM
else:
	import sys

	lowercase  							=					_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





