






"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_snake_case          =   logging.get_logger(__name__)

_snake_case          =   {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_snake_case          =   [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

_snake_case          =   {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
_snake_case          =   {F'''funnel-transformer/{name}''': 512 for name in _model_names}
_snake_case          =   {F'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}






class        UpperCamelCase			(	__UpperCAmelCase      ):
 UpperCamelCase					:       Any			=					VOCAB_FILES_NAMES
 UpperCamelCase					:       int			=					PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase					:       Any			=					PRETRAINED_INIT_CONFIGURATION
 UpperCamelCase					:       Dict			=					FunnelTokenizer
 UpperCamelCase					:       Optional[int]			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase					:       int			=					2


 def __init__(     self		:						Dict							,						UpperCAmelCase__		:						Optional[int]=None							,						UpperCAmelCase__		:						List[Any]=None							,						UpperCAmelCase__		:						Dict=True							,						UpperCAmelCase__		:						List[str]="<unk>"							,						UpperCAmelCase__		:						Optional[Any]="<sep>"							,						UpperCAmelCase__		:						str="<pad>"							,						UpperCAmelCase__		:						List[Any]="<cls>"							,						UpperCAmelCase__		:						Dict="<mask>"							,						UpperCAmelCase__		:						Optional[Any]="<s>"							,						UpperCAmelCase__		:						Dict="</s>"							,						UpperCAmelCase__		:						Optional[Any]=True							,						UpperCAmelCase__		:						Any=True							,						UpperCAmelCase__		:						List[Any]=None							,						UpperCAmelCase__		:						List[str]="##"							,						**UpperCAmelCase__		:						Tuple							,						)					->       Optional[Any]:
   super().__init__(
       UpperCAmelCase__							,						tokenizer_file=UpperCAmelCase__							,						do_lower_case=UpperCAmelCase__							,						unk_token=UpperCAmelCase__							,						sep_token=UpperCAmelCase__							,						pad_token=UpperCAmelCase__							,						cls_token=UpperCAmelCase__							,						mask_token=UpperCAmelCase__							,						bos_token=UpperCAmelCase__							,						eos_token=UpperCAmelCase__							,						clean_text=UpperCAmelCase__							,						tokenize_chinese_chars=UpperCAmelCase__							,						strip_accents=UpperCAmelCase__							,						wordpieces_prefix=UpperCAmelCase__							,						**UpperCAmelCase__							,						)

   _a  :       Optional[Any]             =	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
   if (
       normalizer_state.get("""lowercase"""							,						UpperCAmelCase__							) != do_lower_case
       or normalizer_state.get("""strip_accents"""							,						UpperCAmelCase__							) != strip_accents
       or normalizer_state.get("""handle_chinese_chars"""							,						UpperCAmelCase__							) != tokenize_chinese_chars
   ):
     _a  :       Tuple             =	getattr(UpperCAmelCase__							,						normalizer_state.pop("""type"""							)							)
     _a  :       Dict             =	do_lower_case
     _a  :       Union[str, Any]             =	strip_accents
     _a  :       Dict             =	tokenize_chinese_chars
     _a  :       Dict             =	normalizer_class(**UpperCAmelCase__							)

   _a  :       List[str]             =	do_lower_case


 def _lowercase	(     self		:						List[Any]							,						UpperCAmelCase__		:						str							,						UpperCAmelCase__		:						List[Any]=None							)					->       Tuple:
   _a  :       int             =	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

   return output


 def _lowercase	(     self		:						Dict							,						UpperCAmelCase__		:						int							,						UpperCAmelCase__		:						Dict = None							)					->       Union[str, Any]:
   _a  :       str             =	[self.sep_token_id]
   _a  :       int             =	[self.cls_token_id]
   if token_ids_a is None:
     return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0]
   return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







 def _lowercase	(     self		:						int							,						UpperCAmelCase__		:						Optional[int]							,						UpperCAmelCase__		:						Optional[Any] = None							)					->       Optional[int]:
   _a  :       Union[str, Any]             =	self._tokenizer.model.save(UpperCAmelCase__							,						name=UpperCAmelCase__							)
   return tuple(UpperCAmelCase__							)






'''simple docstring'''




# Author: OMKAR PATHAK, Nwachukwu Chidiebere

# Use a Python dictionary to construct the graph.
from __future__ import annotations

from pprint import pformat
from typing import Generic, TypeVar

lowerCamelCase							:str				    =					TypeVar('''T''')







class 	_lowerCAmelCase							(			Generic[T] ):


	def __init__(self       ,      lowercase = True							):
				A_	:						dict[T, list[T]]	=     {}  # dictionary of lists
				A_	:						Any	=     directed


	def     _a	(self       ,      lowercase       ,      lowercase							):

				if not self.directed:  # For undirected graphs
							# if both source vertex and destination vertex are both present in the
							# adjacency list, add destination vertex to source vertex list of adjacent
							# vertices and add source vertex to destination vertex list of adjacent
							# vertices.
							if source_vertex in self.adj_list and destination_vertex in self.adj_list:
										self.adj_list[source_vertex].append(lowercase							)
										self.adj_list[destination_vertex].append(lowercase							)
							# if only source vertex is present in adjacency list, add destination vertex
							# to source vertex list of adjacent vertices, then create a new vertex with
							# destination vertex as key and assign a list containing the source vertex
							# as it's first adjacent vertex.
							elif source_vertex in self.adj_list:
										self.adj_list[source_vertex].append(lowercase							)
										A_	:						Dict	=     [source_vertex]
							# if only destination vertex is present in adjacency list, add source vertex
							# to destination vertex list of adjacent vertices, then create a new vertex
							# with source vertex as key and assign a list containing the source vertex
							# as it's first adjacent vertex.
							elif destination_vertex in self.adj_list:
										self.adj_list[destination_vertex].append(lowercase							)
										A_	:						int	=     [destination_vertex]
							# if both source vertex and destination vertex are not present in adjacency
							# list, create a new vertex with source vertex as key and assign a list
							# containing the destination vertex as it's first adjacent vertex also
							# create a new vertex with destination vertex as key and assign a list
							# containing the source vertex as it's first adjacent vertex.
							else:
										A_	:						Optional[Any]	=     [destination_vertex]
										A_	:						Tuple	=     [source_vertex]
				else:  # For directed graphs
							# if both source vertex and destination vertex are present in adjacency
							# list, add destination vertex to source vertex list of adjacent vertices.
							if source_vertex in self.adj_list and destination_vertex in self.adj_list:
										self.adj_list[source_vertex].append(lowercase							)
							# if only source vertex is present in adjacency list, add destination
							# vertex to source vertex list of adjacent vertices and create a new vertex
							# with destination vertex as key, which has no adjacent vertex
							elif source_vertex in self.adj_list:
										self.adj_list[source_vertex].append(lowercase							)
										A_	:						Tuple	=     []
							# if only destination vertex is present in adjacency list, create a new
							# vertex with source vertex as key and assign a list containing destination
							# vertex as first adjacent vertex
							elif destination_vertex in self.adj_list:
										A_	:						Tuple	=     [destination_vertex]
							# if both source vertex and destination vertex are not present in adjacency
							# list, create a new vertex with source vertex as key and a list containing
							# destination vertex as it's first adjacent vertex. Then create a new vertex
							# with destination vertex as key, which has no adjacent vertex
							else:
										A_	:						int	=     [destination_vertex]
										A_	:						List[str]	=     []

				return self


	def __repr__(self							):
				return pformat(self.adj_list							)