
'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowercase			:				int    		=			logging.get_logger(__name__)

lowercase			:				List[Any]    		=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase			:				Dict    		=			[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

lowercase			:				Dict    		=			{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
lowercase			:				Union[str, Any]    		=			{f'''funnel-transformer/{name}''': 5_12 for name in _model_names}
lowercase			:				Optional[Any]    		=			{f'''funnel-transformer/{name}''': {'do_lower_case': True} for name in _model_names}





class  A				(							__snake_case						):
     __magic_name__						=		VOCAB_FILES_NAMES
     __magic_name__						=		PRETRAINED_VOCAB_FILES_MAP
     __magic_name__						=		PRETRAINED_INIT_CONFIGURATION
     __magic_name__						=		FunnelTokenizer
     __magic_name__						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__						=		2





     def __init__(							self						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE="<unk>"						,			SCREAMING_SNAKE_CASE="<sep>"						,			SCREAMING_SNAKE_CASE="<pad>"						,			SCREAMING_SNAKE_CASE="<cls>"						,			SCREAMING_SNAKE_CASE="<mask>"						,			SCREAMING_SNAKE_CASE="<s>"						,			SCREAMING_SNAKE_CASE="</s>"						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE="##"						,			**SCREAMING_SNAKE_CASE						,			) ->				Tuple:






         """simple docstring"""






         super().__init__(
             SCREAMING_SNAKE_CASE						,			tokenizer_file=SCREAMING_SNAKE_CASE						,			do_lower_case=SCREAMING_SNAKE_CASE						,			unk_token=SCREAMING_SNAKE_CASE						,			sep_token=SCREAMING_SNAKE_CASE						,			pad_token=SCREAMING_SNAKE_CASE						,			cls_token=SCREAMING_SNAKE_CASE						,			mask_token=SCREAMING_SNAKE_CASE						,			bos_token=SCREAMING_SNAKE_CASE						,			eos_token=SCREAMING_SNAKE_CASE						,			clean_text=SCREAMING_SNAKE_CASE						,			tokenize_chinese_chars=SCREAMING_SNAKE_CASE						,			strip_accents=SCREAMING_SNAKE_CASE						,			wordpieces_prefix=SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE						,			)

         A       :   Tuple					=						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
         if (
             normalizer_state.get('''lowercase'''						,			SCREAMING_SNAKE_CASE	) != do_lower_case
             or normalizer_state.get('''strip_accents'''						,			SCREAMING_SNAKE_CASE	) != strip_accents
             or normalizer_state.get('''handle_chinese_chars'''						,			SCREAMING_SNAKE_CASE	) != tokenize_chinese_chars
         ):
             A       :   List[Any]					=						getattr(SCREAMING_SNAKE_CASE						,			normalizer_state.pop('''type'''	)	)
             A       :   Union[str, Any]					=						do_lower_case
             A       :   Optional[int]					=						strip_accents
             A       :   Union[str, Any]					=						tokenize_chinese_chars
             A       :   Tuple					=						normalizer_class(**SCREAMING_SNAKE_CASE	)

         A       :   int					=						do_lower_case





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=None	) ->				Union[str, Any]:






         """simple docstring"""






         A       :   Optional[Any]					=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

         return output





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				List[int]:






         """simple docstring"""






         A       :   List[Any]					=						[self.sep_token_id]
         A       :   Tuple					=						[self.cls_token_id]
         if token_ids_a is None:
             return len(cls	) * [self.cls_token_type_id] + len(token_ids_a + sep	) * [0]
         return len(cls	) * [self.cls_token_type_id] + len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				Tuple[str]:






         """simple docstring"""






         A       :   Tuple					=						self._tokenizer.model.save(SCREAMING_SNAKE_CASE						,			name=SCREAMING_SNAKE_CASE	)
         return tuple(SCREAMING_SNAKE_CASE	)







# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
  import torch





class  __lowerCAmelCase     (					TensorFormatter[Mapping, """torch.Tensor""", Mapping]							):




  def __init__(				self							,   lowerCAmelCase=None							,   **lowerCAmelCase					)  -> Optional[Any]:
        '''simple docstring'''






        super().__init__(features=lowerCAmelCase					)
        _lowercase	       =torch_tensor_kwargs
        import torch  # noqa import torch at initialization




  def   A__     (				self							,   lowerCAmelCase					)  -> int:
        '''simple docstring'''






        import torch

        if isinstance(lowerCAmelCase							,   lowerCAmelCase					) and column:
              if all(
                  isinstance(lowerCAmelCase							,   torch.Tensor					) and x.shape == column[0].shape and x.dtype == column[0].dtype
                  for x in column					):
                    return torch.stack(lowerCAmelCase					)
        return column




  def   A__     (				self							,   lowerCAmelCase					)  -> str:
        '''simple docstring'''






        import torch

        if isinstance(lowerCAmelCase							,   (str, bytes, type(lowerCAmelCase					))					):
              return value
        elif isinstance(lowerCAmelCase							,   (np.character, np.ndarray)					) and np.issubdtype(value.dtype							,   np.character					):
              return value.tolist()

        _lowercase	       ={}

        if isinstance(lowerCAmelCase							,   (np.number, np.ndarray)					) and np.issubdtype(value.dtype							,   np.integer					):
              _lowercase	       ={'dtype': torch.intaa}
        elif isinstance(lowerCAmelCase							,   (np.number, np.ndarray)					) and np.issubdtype(value.dtype							,   np.floating					):
              _lowercase	       ={'dtype': torch.floataa}
        elif config.PIL_AVAILABLE and "PIL" in sys.modules:
              import PIL.Image

              if isinstance(lowerCAmelCase							,   PIL.Image.Image					):
                    _lowercase	       =np.asarray(lowerCAmelCase					)
        return torch.tensor(lowerCAmelCase							,   **{**default_dtype, **self.torch_tensor_kwargs}					)




  def   A__     (				self							,   lowerCAmelCase					)  -> str:
        '''simple docstring'''






        import torch

        # support for torch, tf, jax etc.
        if hasattr(lowerCAmelCase							,   '__array__'					) and not isinstance(lowerCAmelCase							,   torch.Tensor					):
              _lowercase	       =data_struct.__array__()
        # support for nested types like struct of list of struct
        if isinstance(lowerCAmelCase							,   np.ndarray					):
              if data_struct.dtype == object:  # torch tensors cannot be instantied from an array of objects
                    return self._consolidate([self.recursive_tensorize(lowerCAmelCase					) for substruct in data_struct]					)
        elif isinstance(lowerCAmelCase							,   (list, tuple)					):
              return self._consolidate([self.recursive_tensorize(lowerCAmelCase					) for substruct in data_struct]					)
        return self._tensorize(lowerCAmelCase					)




  def   A__     (				self							,   lowerCAmelCase					)  -> Tuple:
        '''simple docstring'''






        return map_nested(self._recursive_tensorize							,   lowerCAmelCase							,   map_list=lowerCAmelCase					)




  def   A__     (				self							,   lowerCAmelCase					)  -> Mapping:
        '''simple docstring'''






        _lowercase	       =self.numpy_arrow_extractor().extract_row(lowerCAmelCase					)
        _lowercase	       =self.python_features_decoder.decode_row(lowerCAmelCase					)
        return self.recursive_tensorize(lowerCAmelCase					)




  def   A__     (				self							,   lowerCAmelCase					)  -> "torch.Tensor":
        '''simple docstring'''






        _lowercase	       =self.numpy_arrow_extractor().extract_column(lowerCAmelCase					)
        _lowercase	       =self.python_features_decoder.decode_column(lowerCAmelCase							,   pa_table.column_names[0]					)
        _lowercase	       =self.recursive_tensorize(lowerCAmelCase					)
        _lowercase	       =self._consolidate(lowerCAmelCase					)
        return column




  def   A__     (				self							,   lowerCAmelCase					)  -> Mapping:
        '''simple docstring'''






        _lowercase	       =self.numpy_arrow_extractor().extract_batch(lowerCAmelCase					)
        _lowercase	       =self.python_features_decoder.decode_batch(lowerCAmelCase					)
        _lowercase	       =self.recursive_tensorize(lowerCAmelCase					)
        for column_name in batch:
              _lowercase	       =self._consolidate(batch[column_name]					)
        return batch


