


"""simple docstring"""







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_fnet import FNetTokenizer
else:
		a__				:							Union[str, Any]						=     None

a__				:							Optional[Any]						=     logging.get_logger(__name__)
a__				:							Optional[int]						=     {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

a__				:							Optional[int]						=     {
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json''',
    },
}

a__				:							List[str]						=     {
    '''google/fnet-base''': 5_1_2,
    '''google/fnet-large''': 5_1_2,
}

a__				:							Union[str, Any]						=     '''▁'''




class  UpperCamelCase_	( UpperCamelCase):

	"""simple docstring"""





	snake_case__				:  List[Any]    					=			VOCAB_FILES_NAMES
	snake_case__				:  int    					=			PRETRAINED_VOCAB_FILES_MAP
	snake_case__				:  Dict    					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__				:  List[str]    					=			["input_ids", "token_type_ids"]
	snake_case__				:  Tuple    					=			FNetTokenizer




	def __init__( self			:				str			,			UpperCAmelCase__			:				Optional[int]=None			,			UpperCAmelCase__			:				Optional[int]=None			,			UpperCAmelCase__			:				Optional[Any]=False			,			UpperCAmelCase__			:				Any=True			,			UpperCAmelCase__			:				int=True			,			UpperCAmelCase__			:				List[Any]="<unk>"			,			UpperCAmelCase__			:				Dict="[SEP]"			,			UpperCAmelCase__			:				Tuple="<pad>"			,			UpperCAmelCase__			:				Any="[CLS]"			,			UpperCAmelCase__			:				List[Any]="[MASK]"			,			**UpperCAmelCase__			:				Union[str, Any]			,			)       ->	str:
						# Mask token behave like a normal word, i.e. include the space before it and
						# is included in the raw text, there should be a match in a non-normalized sentence.
						__SCREAMING_SNAKE_CASE						   =				(
						    AddedToken(UpperCAmelCase__			,			lstrip=UpperCAmelCase__			,			rstrip=UpperCAmelCase__			,			normalized=UpperCAmelCase__   )
						    if isinstance(UpperCAmelCase__			,			UpperCAmelCase__   )
						    else mask_token
						)

						super().__init__(
						    UpperCAmelCase__			,			tokenizer_file=UpperCAmelCase__			,			do_lower_case=UpperCAmelCase__			,			remove_space=UpperCAmelCase__			,			keep_accents=UpperCAmelCase__			,			unk_token=UpperCAmelCase__			,			sep_token=UpperCAmelCase__			,			pad_token=UpperCAmelCase__			,			cls_token=UpperCAmelCase__			,			mask_token=UpperCAmelCase__			,			**UpperCAmelCase__			,			)

						__SCREAMING_SNAKE_CASE						   =				do_lower_case
						__SCREAMING_SNAKE_CASE						   =				remove_space
						__SCREAMING_SNAKE_CASE						   =				keep_accents
						__SCREAMING_SNAKE_CASE						   =				vocab_file
						__SCREAMING_SNAKE_CASE						   =				False if not self.vocab_file else True




	def   UpperCAmelCase_      ( self			:				List[Any]			,			UpperCAmelCase__			:				List[int]			,			UpperCAmelCase__			:				Optional[List[int]] = None   )       ->	List[int]:
						__SCREAMING_SNAKE_CASE						   =				[self.sep_token_id]
						__SCREAMING_SNAKE_CASE						   =				[self.cls_token_id]
						if token_ids_a is None:
											return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep




	def   UpperCAmelCase_      ( self			:				int			,			UpperCAmelCase__			:				List[int]			,			UpperCAmelCase__			:				Optional[List[int]] = None   )       ->	List[int]:
						__SCREAMING_SNAKE_CASE						   =				[self.sep_token_id]
						__SCREAMING_SNAKE_CASE						   =				[self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep   ) * [0]
						return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




	def   UpperCAmelCase_      ( self			:				str			,			UpperCAmelCase__			:				str			,			UpperCAmelCase__			:				Optional[str] = None   )       ->	Tuple[str]:
						if not os.path.isdir(UpperCAmelCase__   ):
											logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
											return
						__SCREAMING_SNAKE_CASE						   =				os.path.join(
						    UpperCAmelCase__			,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )

						if os.path.abspath(self.vocab_file   ) != os.path.abspath(UpperCAmelCase__   ):
											copyfile(self.vocab_file			,			UpperCAmelCase__   )

						return (out_vocab_file,)





"""simple docstring"""







from ..utils import DummyObject, requires_backends







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      List[Any]    ,			*SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			**SCREAMING_SNAKE_CASE_				:      Any       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Any    ,			*SCREAMING_SNAKE_CASE_				:      int    ,			**SCREAMING_SNAKE_CASE_				:      List[str]       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      int    ,			*SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			**SCREAMING_SNAKE_CASE_				:      Optional[int]       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      Tuple    ,			*SCREAMING_SNAKE_CASE_				:      Tuple    ,			**SCREAMING_SNAKE_CASE_				:      Any       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Union[str, Any]    ,			*SCREAMING_SNAKE_CASE_				:      Dict    ,			**SCREAMING_SNAKE_CASE_				:      Dict       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Dict    ,			*SCREAMING_SNAKE_CASE_				:      List[str]    ,			**SCREAMING_SNAKE_CASE_				:      Optional[Any]       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      int    ,			*SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			**SCREAMING_SNAKE_CASE_				:      List[str]       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      List[Any]    ,			*SCREAMING_SNAKE_CASE_				:      Any    ,			**SCREAMING_SNAKE_CASE_				:      List[Any]       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Union[str, Any]    ,			*SCREAMING_SNAKE_CASE_				:      List[str]    ,			**SCREAMING_SNAKE_CASE_				:      List[str]       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      Tuple    ,			*SCREAMING_SNAKE_CASE_				:      List[str]    ,			**SCREAMING_SNAKE_CASE_				:      int       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Tuple    ,			*SCREAMING_SNAKE_CASE_				:      List[str]    ,			**SCREAMING_SNAKE_CASE_				:      str       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      List[Any]    ,			*SCREAMING_SNAKE_CASE_				:      Dict    ,			**SCREAMING_SNAKE_CASE_				:      int       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      List[str]    ,			*SCREAMING_SNAKE_CASE_				:      int    ,			**SCREAMING_SNAKE_CASE_				:      int       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Optional[Any]    ,			*SCREAMING_SNAKE_CASE_				:      str    ,			**SCREAMING_SNAKE_CASE_				:      List[str]       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      List[str]    ,			*SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			**SCREAMING_SNAKE_CASE_				:      Dict       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      Any    ,			*SCREAMING_SNAKE_CASE_				:      int    ,			**SCREAMING_SNAKE_CASE_				:      Dict       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      str    ,			*SCREAMING_SNAKE_CASE_				:      Optional[Any]    ,			**SCREAMING_SNAKE_CASE_				:      Any       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Tuple    ,			*SCREAMING_SNAKE_CASE_				:      List[Any]    ,			**SCREAMING_SNAKE_CASE_				:      Union[str, Any]       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      Tuple    ,			*SCREAMING_SNAKE_CASE_				:      Any    ,			**SCREAMING_SNAKE_CASE_				:      Optional[Any]       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Optional[int]    ,			*SCREAMING_SNAKE_CASE_				:      int    ,			**SCREAMING_SNAKE_CASE_				:      Dict       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Optional[int]    ,			*SCREAMING_SNAKE_CASE_				:      Tuple    ,			**SCREAMING_SNAKE_CASE_				:      Union[str, Any]       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      int    ,			*SCREAMING_SNAKE_CASE_				:      int    ,			**SCREAMING_SNAKE_CASE_				:      Union[str, Any]       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Union[str, Any]    ,			*SCREAMING_SNAKE_CASE_				:      List[str]    ,			**SCREAMING_SNAKE_CASE_				:      Tuple       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Dict    ,			*SCREAMING_SNAKE_CASE_				:      List[Any]    ,			**SCREAMING_SNAKE_CASE_				:      Tuple       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      Union[str, Any]    ,			*SCREAMING_SNAKE_CASE_				:      int    ,			**SCREAMING_SNAKE_CASE_				:      Dict       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Any    ,			*SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			**SCREAMING_SNAKE_CASE_				:      str       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Dict    ,			*SCREAMING_SNAKE_CASE_				:      List[Any]    ,			**SCREAMING_SNAKE_CASE_				:      Tuple       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      Dict    ,			*SCREAMING_SNAKE_CASE_				:      List[Any]    ,			**SCREAMING_SNAKE_CASE_				:      List[Any]       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Optional[int]    ,			*SCREAMING_SNAKE_CASE_				:      Tuple    ,			**SCREAMING_SNAKE_CASE_				:      List[Any]       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Dict    ,			*SCREAMING_SNAKE_CASE_				:      Optional[Any]    ,			**SCREAMING_SNAKE_CASE_				:      int       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      Optional[int]    ,			*SCREAMING_SNAKE_CASE_				:      List[Any]    ,			**SCREAMING_SNAKE_CASE_				:      Dict       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Optional[Any]    ,			*SCREAMING_SNAKE_CASE_				:      Union[str, Any]    ,			**SCREAMING_SNAKE_CASE_				:      Optional[Any]       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Union[str, Any]    ,			*SCREAMING_SNAKE_CASE_				:      Optional[Any]    ,			**SCREAMING_SNAKE_CASE_				:      int       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      Tuple    ,			*SCREAMING_SNAKE_CASE_				:      Any    ,			**SCREAMING_SNAKE_CASE_				:      Any       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Union[str, Any]    ,			*SCREAMING_SNAKE_CASE_				:      Tuple    ,			**SCREAMING_SNAKE_CASE_				:      List[str]       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Tuple    ,			*SCREAMING_SNAKE_CASE_				:      Optional[Any]    ,			**SCREAMING_SNAKE_CASE_				:      str       ):
        requires_backends(cls    ,			['flax']       )







class       UpperCamelCase__   (	metaclass=lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     ["""flax"""]



      def __init__(      self				:      Optional[Any]    ,			*SCREAMING_SNAKE_CASE_				:      Any    ,			**SCREAMING_SNAKE_CASE_				:      int       ):
        requires_backends(self    ,			['flax']       )



      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Tuple    ,			*SCREAMING_SNAKE_CASE_				:      Dict    ,			**SCREAMING_SNAKE_CASE_				:      Optional[int]       ):
        requires_backends(cls    ,			['flax']       )





      @classmethod
      def      SCREAMING_SNAKE_CASE__ (      cls				:      Optional[int]    ,			*SCREAMING_SNAKE_CASE_				:      str    ,			**SCREAMING_SNAKE_CASE_				:      Tuple       ):
        requires_backends(cls    ,			['flax']       )





