



"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCAmelCase_   						=   logging.get_logger(__name__)

lowerCAmelCase_   						=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase_   						=   [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

lowerCAmelCase_   						=   {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
lowerCAmelCase_   						=   {F'''funnel-transformer/{name}''': 512 for name in _model_names}
lowerCAmelCase_   						=   {F'''funnel-transformer/{name}''': {'do_lower_case': True} for name in _model_names}
class        __A       (	A_    ):







		'''simple docstring'''





		lowerCAmelCase			:					str   	=				VOCAB_FILES_NAMES
		lowerCAmelCase			:					List[str]   	=				PRETRAINED_VOCAB_FILES_MAP
		lowerCAmelCase			:					int   	=				PRETRAINED_INIT_CONFIGURATION
		lowerCAmelCase			:					Dict   	=				FunnelTokenizer
		lowerCAmelCase			:					Tuple   	=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowerCAmelCase			:					int   	=				2







		def __init__(	self						:  Optional[int]			,_snake_case						:  Dict=None			,_snake_case						:  List[str]=None			,_snake_case						:  Optional[Any]=True			,_snake_case						:  Optional[int]="<unk>"			,_snake_case						:  Dict="<sep>"			,_snake_case						:  Any="<pad>"			,_snake_case						:  str="<cls>"			,_snake_case						:  Optional[Any]="<mask>"			,_snake_case						:  int="<s>"			,_snake_case						:  Dict="</s>"			,_snake_case						:  Optional[int]=True			,_snake_case						:  List[str]=True			,_snake_case						:  Dict=None			,_snake_case						:  str="##"			,**_snake_case						:  Optional[Any]			,)					->		Any:






				"""simple docstring"""



				super().__init__(
				    _snake_case			,tokenizer_file=_snake_case			,do_lower_case=_snake_case			,unk_token=_snake_case			,sep_token=_snake_case			,pad_token=_snake_case			,cls_token=_snake_case			,mask_token=_snake_case			,bos_token=_snake_case			,eos_token=_snake_case			,clean_text=_snake_case			,tokenize_chinese_chars=_snake_case			,strip_accents=_snake_case			,wordpieces_prefix=_snake_case			,**_snake_case			,)

				lowercase__   :    Union[str, Any]				=			json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
				if (
				    normalizer_state.get('''lowercase'''			,_snake_case						) != do_lower_case
				    or normalizer_state.get('''strip_accents'''			,_snake_case						) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''			,_snake_case						) != tokenize_chinese_chars
				):
						lowercase__   :    List[str]				=			getattr(_snake_case			,normalizer_state.pop('''type'''						)						)
						lowercase__   :    List[str]				=			do_lower_case
						lowercase__   :    Any				=			strip_accents
						lowercase__   :    Union[str, Any]				=			tokenize_chinese_chars
						lowercase__   :    Union[str, Any]				=			normalizer_class(**_snake_case						)

				lowercase__   :    List[str]				=			do_lower_case







		def 					UpperCAmelCase    (	self						:  int			,_snake_case						:  Tuple			,_snake_case						:  Tuple=None						)					->		Optional[Any]:






				"""simple docstring"""



				lowercase__   :    Any				=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

				return output







		def 					UpperCAmelCase    (	self						:  Tuple			,_snake_case						:  List[int]			,_snake_case						:  Optional[List[int]] = None						)					->		List[int]:






				"""simple docstring"""



				lowercase__   :    Optional[int]				=			[self.sep_token_id]
				lowercase__   :    List[Any]				=			[self.cls_token_id]
				if token_ids_a is None:
						return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
				return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







		def 					UpperCAmelCase    (	self						:  Tuple			,_snake_case						:  str			,_snake_case						:  Optional[str] = None						)					->		Tuple[str]:






				"""simple docstring"""



				lowercase__   :    str				=			self._tokenizer.model.save(_snake_case			,name=_snake_case						)
				return tuple(_snake_case						)


import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


UpperCAmelCase	      = logging.get_logger(__name__)






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE							):
	lowercase       =    r'\w+[.]\d+'
	lowercase       =    re.findall(__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE							)
	for pat in pats:
		lowercase       =    key.replace(__SCREAMING_SNAKE_CASE ,							'_'.join(pat.split('.'							)							)							)
	return key






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE							):
	lowercase       =    pt_tuple_key[:-1] + ('scale',)
	if (
	    any('norm' in str_ for str_ in pt_tuple_key							)
	    and (pt_tuple_key[-1] == "bias")
	    and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
	    and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
	):
		lowercase       =    pt_tuple_key[:-1] + ('scale',)
		return renamed_pt_tuple_key, pt_tensor
	elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
		lowercase       =    pt_tuple_key[:-1] + ('scale',)
		return renamed_pt_tuple_key, pt_tensor

	# embedding
	if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
		lowercase       =    pt_tuple_key[:-1] + ('embedding',)
		return renamed_pt_tuple_key, pt_tensor

	# conv layer
	lowercase       =    pt_tuple_key[:-1] + ('kernel',)
	if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
		lowercase       =    pt_tensor.transpose(2 ,							3 ,							1 ,							0							)
		return renamed_pt_tuple_key, pt_tensor

	# linear layer
	lowercase       =    pt_tuple_key[:-1] + ('kernel',)
	if pt_tuple_key[-1] == "weight":
		lowercase       =    pt_tensor.T
		return renamed_pt_tuple_key, pt_tensor

	# old PyTorch layer norm weight
	lowercase       =    pt_tuple_key[:-1] + ('weight',)
	if pt_tuple_key[-1] == "gamma":
		return renamed_pt_tuple_key, pt_tensor

	# old PyTorch layer norm bias
	lowercase       =    pt_tuple_key[:-1] + ('bias',)
	if pt_tuple_key[-1] == "beta":
		return renamed_pt_tuple_key, pt_tensor

	return pt_tuple_key, pt_tensor






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE=42							):
	# Step 1: Convert pytorch tensor to numpy
	lowercase       =    {k: v.numpy() for k, v in pt_state_dict.items()}

	# Step 2: Since the model is stateless, get random Flax params
	lowercase       =    flax_model.init_weights(PRNGKey(__SCREAMING_SNAKE_CASE							)							)

	lowercase       =    flatten_dict(__SCREAMING_SNAKE_CASE							)
	lowercase       =    {}

	# Need to change some parameters name to match Flax names
	for pt_key, pt_tensor in pt_state_dict.items():
		lowercase       =    rename_key(__SCREAMING_SNAKE_CASE							)
		lowercase       =    tuple(renamed_pt_key.split('.'							)							)

		# Correctly rename weight parameters
		lowercase  ,					lowercase       =    rename_key_and_reshape_tensor(__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE							)

		if flax_key in random_flax_state_dict:
			if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
				raise ValueError(
				    F'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
				    F'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''							)

        # also add unexpected weight so that warning is thrown
		lowercase       =    jnp.asarray(__SCREAMING_SNAKE_CASE							)

	return unflatten_dict(__SCREAMING_SNAKE_CASE							)

