






'''simple docstring'''






import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
      import torch

      from ..models.auto.modeling_auto import (
          MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
          MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
      )

if is_tf_available() and is_tensorflow_probability_available():
      import tensorflow as tf
      import tensorflow_probability as tfp

      from ..models.auto.modeling_tf_auto import (
          TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
          TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
      )




class a_     (     UpperCamelCase_				):







   def __call__(self       , __a=None       , __a=None       , **__a)      ->   str:
      """simple docstring"""






      requires_backends(self       , 'pandas')
      import pandas as pd

      if table is None:
         raise ValueError('Keyword argument `table` cannot be None.')
      elif query is None:
         if isinstance(__a       , __a) and table.get('query') is not None and table.get('table') is not None:
            __snake_case					:   Dict  				=			[table]
         elif isinstance(__a       , __a) and len(__a) > 0:
            if not all(isinstance(__a       , __a) for d in table):
               raise ValueError(
                   F"""Keyword argument `table` should be a list of dict, but is {(type(__a) for d in table)}""")

            if table[0].get('query') is not None and table[0].get('table') is not None:
               __snake_case					:   Optional[Any]  				=			table
            else:
               raise ValueError(
                   'If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`'
                   F""" and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys.""")
         elif Dataset is not None and isinstance(__a       , __a) or isinstance(__a       , types.GeneratorType):
            return table
         else:
            raise ValueError(
                'Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but '
                F"""is {type(__a)})""")
      else:
         __snake_case					:   List[Any]  				=			[{'table': table, 'query': query}]

      for tqa_pipeline_input in tqa_pipeline_inputs:
         if not isinstance(tqa_pipeline_input['table']       , pd.DataFrame):
            if tqa_pipeline_input["table"] is None:
               raise ValueError('Table cannot be None.')

            __snake_case					:   Dict  				=			pd.DataFrame(tqa_pipeline_input['table'])

      return tqa_pipeline_inputs




@add_end_docstrings(UpperCamelCase_				)
class a_     (     UpperCamelCase_				):
   _snake_case	   =				"""table,query"""







   def __init__(self       , __a=TableQuestionAnsweringArgumentHandler()       , *__a       , **__a)      ->   int:
      """simple docstring"""






      super().__init__(*__a       , **__a)
      __snake_case					:   List[Any]  				=			args_parser

      self.check_model_type(
          dict(
              TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
              + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items())
          if self.framework == 'tf'
          else dict(
              MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()))

      __snake_case					:   List[Any]  				=			bool(getattr(self.model.config       , 'aggregation_labels'       , __a)) and bool(
          getattr(self.model.config       , 'num_aggregation_labels'       , __a))
      __snake_case					:   Dict  				=			'tapas' if hasattr(self.model.config       , 'aggregation_labels') else None







   def      SCREAMING_SNAKE_CASE__       (self       , **__a)      ->   Dict:
      """simple docstring"""






      return self.model(**__a)







   def      SCREAMING_SNAKE_CASE__       (self       , **__a)      ->   str:
      """simple docstring"""






      if self.framework == "pt":
         __snake_case					:   List[Any]  				=			[]
         __snake_case					:   List[Any]  				=			[]
         __snake_case					:   List[Any]  				=			None
         __snake_case					:   Any  				=			inputs['input_ids'].shape[0]

         __snake_case					:   List[Any]  				=			inputs['input_ids'].to(self.device)
         __snake_case					:   Optional[Any]  				=			inputs['attention_mask'].to(self.device)
         __snake_case					:   Dict  				=			inputs['token_type_ids'].to(self.device)
         __snake_case					:   str  				=			None

         for index in range(__a):
            # If sequences have already been processed, the token type IDs will be created according to the previous
            # answer.
            if prev_answers is not None:
               __snake_case					:   Dict  				=			token_type_ids_example[:, 3]  # shape (seq_len,)
               __snake_case					:   List[Any]  				=			np.zeros_like(prev_labels_example.cpu().numpy())  # shape (seq_len,)

               __snake_case					:   List[Any]  				=			token_type_ids[index]  # shape (seq_len, 7)
               for i in range(model_labels.shape[0]):
                  __snake_case					:   Union[str, Any]  				=			token_type_ids_example[:, 0].tolist()[i]
                  __snake_case					:   str  				=			token_type_ids_example[:, 1].tolist()[i] - 1
                  __snake_case					:   Optional[int]  				=			token_type_ids_example[:, 2].tolist()[i] - 1

                  if row_id >= 0 and col_id >= 0 and segment_id == 1:
                     __snake_case					:   int  				=			int(prev_answers[(col_id, row_id)])

               __snake_case					:   Any  				=			torch.from_numpy(__a).type(torch.long).to(self.device)

            __snake_case					:   Dict  				=			input_ids[index]
            __snake_case					:   str  				=			attention_mask[index]  # shape (seq_len,)
            __snake_case					:   Optional[Any]  				=			token_type_ids[index]  # shape (seq_len, 7)
            __snake_case					:   Optional[int]  				=			self.model(
                input_ids=input_ids_example.unsqueeze(0)       , attention_mask=attention_mask_example.unsqueeze(0)       , token_type_ids=token_type_ids_example.unsqueeze(0)       , )
            __snake_case					:   Dict  				=			outputs.logits

            if self.aggregate:
               all_aggregations.append(outputs.logits_aggregation)

            all_logits.append(__a)

            __snake_case					:   int  				=			torch.distributions.Bernoulli(logits=__a)
            __snake_case					:   str  				=			dist_per_token.probs * attention_mask_example.type(torch.floataa).to(
                dist_per_token.probs.device)

            __snake_case					:   str  				=			collections.defaultdict(__a)
            for i, p in enumerate(probabilities.squeeze().tolist()):
               __snake_case					:   List[Any]  				=			token_type_ids_example[:, 0].tolist()[i]
               __snake_case					:   int  				=			token_type_ids_example[:, 1].tolist()[i] - 1
               __snake_case					:   Optional[int]  				=			token_type_ids_example[:, 2].tolist()[i] - 1
               if col >= 0 and row >= 0 and segment_id == 1:
                  coords_to_probs[(col, row)].append(__a)

            __snake_case					:   List[str]  				=			{key: np.array(coords_to_probs[key]).mean() > 0.5 for key in coords_to_probs}

         __snake_case					:   Dict  				=			torch.cat(tuple(__a)       , 0)

         return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(__a)       , 0))
      else:
         __snake_case					:   Optional[Any]  				=			[]
         __snake_case					:   Optional[Any]  				=			[]
         __snake_case					:   Union[str, Any]  				=			None
         __snake_case					:   Union[str, Any]  				=			inputs['input_ids'].shape[0]

         __snake_case					:   List[Any]  				=			inputs['input_ids']
         __snake_case					:   Optional[int]  				=			inputs['attention_mask']
         __snake_case					:   Union[str, Any]  				=			inputs['token_type_ids'].numpy()
         __snake_case					:   Optional[Any]  				=			None

         for index in range(__a):
            # If sequences have already been processed, the token type IDs will be created according to the previous
            # answer.
            if prev_answers is not None:
               __snake_case					:   Tuple  				=			token_type_ids_example[:, 3]  # shape (seq_len,)
               __snake_case					:   str  				=			np.zeros_like(__a       , dtype=np.intaa)  # shape (seq_len,)

               __snake_case					:   int  				=			token_type_ids[index]  # shape (seq_len, 7)
               for i in range(model_labels.shape[0]):
                  __snake_case					:   Any  				=			token_type_ids_example[:, 0].tolist()[i]
                  __snake_case					:   int  				=			token_type_ids_example[:, 1].tolist()[i] - 1
                  __snake_case					:   Dict  				=			token_type_ids_example[:, 2].tolist()[i] - 1

                  if row_id >= 0 and col_id >= 0 and segment_id == 1:
                     __snake_case					:   str  				=			int(prev_answers[(col_id, row_id)])

               __snake_case					:   Any  				=			model_labels

            __snake_case					:   Union[str, Any]  				=			input_ids[index]
            __snake_case					:   Any  				=			attention_mask[index]  # shape (seq_len,)
            __snake_case					:   Dict  				=			token_type_ids[index]  # shape (seq_len, 7)
            __snake_case					:   Dict  				=			self.model(
                input_ids=np.expand_dims(__a       , axis=0)       , attention_mask=np.expand_dims(__a       , axis=0)       , token_type_ids=np.expand_dims(__a       , axis=0)       , )
            __snake_case					:   Union[str, Any]  				=			outputs.logits

            if self.aggregate:
               all_aggregations.append(outputs.logits_aggregation)

            all_logits.append(__a)

            __snake_case					:   int  				=			tfp.distributions.Bernoulli(logits=__a)
            __snake_case					:   List[Any]  				=			dist_per_token.probs_parameter() * tf.cast(__a       , tf.floataa)

            __snake_case					:   int  				=			collections.defaultdict(__a)
            __snake_case					:   Optional[Any]  				=			token_type_ids_example
            for i, p in enumerate(tf.squeeze(__a).numpy().tolist()):
               __snake_case					:   Optional[int]  				=			token_type_ids_example[:, 0].tolist()[i]
               __snake_case					:   str  				=			token_type_ids_example[:, 1].tolist()[i] - 1
               __snake_case					:   Dict  				=			token_type_ids_example[:, 2].tolist()[i] - 1
               if col >= 0 and row >= 0 and segment_id == 1:
                  coords_to_probs[(col, row)].append(__a)

            __snake_case					:   Tuple  				=			{key: np.array(coords_to_probs[key]).mean() > 0.5 for key in coords_to_probs}

         __snake_case					:   int  				=			tf.concat(tuple(__a)       , 0)

         return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(__a)       , 0))







   def __call__(self       , *__a       , **__a)      ->   List[Any]:
      """simple docstring"""






      __snake_case					:   Any  				=			self._args_parser(*__a       , **__a)

      __snake_case					:   str  				=			super().__call__(__a       , **__a)
      if len(__a) == 1:
         return results[0]
      return results







   def      SCREAMING_SNAKE_CASE__       (self       , __a=None       , __a=None       , __a=None       , **__a)      ->   Any:
      """simple docstring"""






      __snake_case					:   str  				=			{}
      if padding is not None:
         __snake_case					:   Dict  				=			padding
      if truncation is not None:
         __snake_case					:   List[Any]  				=			truncation

      __snake_case					:   List[Any]  				=			{}
      if sequential is not None:
         __snake_case					:   Any  				=			sequential
      return preprocess_params, forward_params, {}







   def      SCREAMING_SNAKE_CASE__       (self       , __a       , __a=None       , __a=True       , __a=None)      ->   Any:
      """simple docstring"""






      if truncation is None:
         if self.type == "tapas":
            __snake_case					:   int  				=			'drop_rows_to_fit'
         else:
            __snake_case					:   Union[str, Any]  				=			'do_not_truncate'

      __snake_case				,__snake_case					:   Optional[Any]  				=			pipeline_input['table'], pipeline_input['query']
      if table.empty:
         raise ValueError('table is empty')
      if query is None or query == "":
         raise ValueError('query is empty')
      __snake_case					:   Optional[Any]  				=			self.tokenizer(__a       , __a       , return_tensors=self.framework       , truncation=__a       , padding=__a)
      __snake_case					:   str  				=			table
      return inputs







   def      SCREAMING_SNAKE_CASE__       (self       , __a       , __a=False)      ->   Union[str, Any]:
      """simple docstring"""






      __snake_case					:   List[Any]  				=			model_inputs.pop('table')

      if self.type == "tapas":
         if sequential:
            __snake_case					:   Dict  				=			self.sequential_inference(**__a)
         else:
            __snake_case					:   List[Any]  				=			self.batch_inference(**__a)
      else:
         __snake_case					:   Optional[Any]  				=			self.model.generate(**__a)
      __snake_case					:   Optional[Any]  				=			{'model_inputs': model_inputs, 'table': table, 'outputs': outputs}
      return model_outputs







   def      SCREAMING_SNAKE_CASE__       (self       , __a)      ->   int:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			model_outputs['model_inputs']
      __snake_case					:   Dict  				=			model_outputs['table']
      __snake_case					:   Tuple  				=			model_outputs['outputs']
      if self.type == "tapas":
         if self.aggregate:
            __snake_case				,__snake_case					:   Dict  				=			outputs[:2]
            __snake_case					:   Tuple  				=			self.tokenizer.convert_logits_to_predictions(__a       , __a       , __a)
            __snake_case				,__snake_case					:   Tuple  				=			predictions
            __snake_case					:   List[str]  				=			{i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(__a)}

            __snake_case					:   List[str]  				=			self.model.config.no_aggregation_label_index
            __snake_case					:   List[str]  				=			{
                i: aggregators[i] + ' > ' for i, pred in enumerate(__a) if pred != no_agg_label_index
            }
         else:
            __snake_case					:   Any  				=			outputs[0]
            __snake_case					:   str  				=			self.tokenizer.convert_logits_to_predictions(__a       , __a)
            __snake_case					:   List[str]  				=			predictions[0]
            __snake_case					:   Optional[int]  				=			{}
            __snake_case					:   Dict  				=			{}
         __snake_case					:   Optional[int]  				=			[]
         for index, coordinates in enumerate(__a):
            __snake_case					:   List[str]  				=			[table.iat[coordinate] for coordinate in coordinates]
            __snake_case					:   Optional[int]  				=			aggregators.get(__a       , '')
            __snake_case					:   Tuple  				=			aggregators_prefix.get(__a       , '')
            __snake_case					:   List[str]  				=			{
                'answer': aggregator_prefix + ', '.join(__a),
                'coordinates': coordinates,
                'cells': [table.iat[coordinate] for coordinate in coordinates],
            }
            if aggregator:
               __snake_case					:   Dict  				=			aggregator

            answers.append(__a)
         if len(__a) == 0:
            raise PipelineException('Empty answer')
      else:
         __snake_case					:   Dict  				=			[{'answer': answer} for answer in self.tokenizer.batch_decode(__a       , skip_special_tokens=__a)]

      return answers if len(__a) > 1 else answers[0]







'''simple docstring'''






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class a_     (     UpperCamelCase_		,   unittest.TestCase				):
   _snake_case	   =				ElectraTokenizer
   _snake_case	   =				ElectraTokenizerFast
   _snake_case	   =				True
   _snake_case	   =				True
   _snake_case	   =				filter_non_english







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      super().setUp()

      __snake_case					:   List[Any]  				=			[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      __snake_case					:   int  				=			os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES['vocab_file'])
      with open(self.vocab_file       , 'w'       , encoding='utf-8') as vocab_writer:
         vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))







   def      SCREAMING_SNAKE_CASE__       (self       , __a)      ->   int:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			'UNwant\u00E9d,running'
      __snake_case					:   Tuple  				=			'unwanted, running'
      return input_text, output_text







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			self.tokenizer_class(self.vocab_file)

      __snake_case					:   str  				=			tokenizer.tokenize('UNwant\u00E9d,running')
      self.assertListEqual(__a       , ['un', '##want', '##ed', ',', 'runn', '##ing'])
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__a)       , [9, 6, 7, 1_2, 1_0, 1_1])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      if not self.test_rust_tokenizer:
         return

      __snake_case					:   Union[str, Any]  				=			self.get_tokenizer()
      __snake_case					:   Dict  				=			self.get_rust_tokenizer()

      __snake_case					:   Dict  				=			'UNwant\u00E9d,running'

      __snake_case					:   Tuple  				=			tokenizer.tokenize(__a)
      __snake_case					:   List[Any]  				=			rust_tokenizer.tokenize(__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   Any  				=			tokenizer.encode(__a       , add_special_tokens=__a)
      __snake_case					:   int  				=			rust_tokenizer.encode(__a       , add_special_tokens=__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[str]  				=			self.get_rust_tokenizer()
      __snake_case					:   Optional[int]  				=			tokenizer.encode(__a)
      __snake_case					:   Dict  				=			rust_tokenizer.encode(__a)
      self.assertListEqual(__a       , __a)

      # With lower casing
      __snake_case					:   Union[str, Any]  				=			self.get_tokenizer(do_lower_case=__a)
      __snake_case					:   List[Any]  				=			self.get_rust_tokenizer(do_lower_case=__a)

      __snake_case					:   List[str]  				=			'UNwant\u00E9d,running'

      __snake_case					:   List[Any]  				=			tokenizer.tokenize(__a)
      __snake_case					:   Union[str, Any]  				=			rust_tokenizer.tokenize(__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[str]  				=			tokenizer.encode(__a       , add_special_tokens=__a)
      __snake_case					:   int  				=			rust_tokenizer.encode(__a       , add_special_tokens=__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[Any]  				=			self.get_rust_tokenizer()
      __snake_case					:   Optional[Any]  				=			tokenizer.encode(__a)
      __snake_case					:   Tuple  				=			rust_tokenizer.encode(__a)
      self.assertListEqual(__a       , __a)







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   int  				=			BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')       , ['ah', '\u535A', '\u63A8', 'zz'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   Optional[Any]  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')       , ['hello', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   Any  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hällo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['h\u00E9llo'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Optional[Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Tuple  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Union[str, Any]:
      """simple docstring"""






      __snake_case					:   Dict  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')       , ['HeLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['HäLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   List[Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['HaLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			BasicTokenizer(do_lower_case=__a       , never_split=['[UNK]'])

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')       , ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      __snake_case					:   Tuple  				=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      __snake_case					:   List[str]  				=			{}
      for i, token in enumerate(__a):
         __snake_case					:   str  				=			i
      __snake_case					:   List[str]  				=			WordpieceTokenizer(vocab=__a       , unk_token='[UNK]')

      self.assertListEqual(tokenizer.tokenize('')       , [])

      self.assertListEqual(tokenizer.tokenize('unwanted running')       , ['un', '##want', '##ed', 'runn', '##ing'])

      self.assertListEqual(tokenizer.tokenize('unwantedX running')       , ['[UNK]', 'runn', '##ing'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      self.assertTrue(_is_whitespace(' '))
      self.assertTrue(_is_whitespace('\t'))
      self.assertTrue(_is_whitespace('\r'))
      self.assertTrue(_is_whitespace('\n'))
      self.assertTrue(_is_whitespace('\u00A0'))

      self.assertFalse(_is_whitespace('A'))
      self.assertFalse(_is_whitespace('-'))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      self.assertTrue(_is_control('\u0005'))

      self.assertFalse(_is_control('A'))
      self.assertFalse(_is_control(' '))
      self.assertFalse(_is_control('\t'))
      self.assertFalse(_is_control('\r'))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      self.assertTrue(_is_punctuation('-'))
      self.assertTrue(_is_punctuation('$'))
      self.assertTrue(_is_punctuation('`'))
      self.assertTrue(_is_punctuation('.'))

      self.assertFalse(_is_punctuation('A'))
      self.assertFalse(_is_punctuation(' '))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Optional[Any]:
      """simple docstring"""






      __snake_case					:   Dict  				=			self.get_tokenizer()
      __snake_case					:   Dict  				=			self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]       , [['[UNK]'], [], ['[UNK]']])

      self.assertListEqual(
          [rust_tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]       , [['[UNK]'], [], ['[UNK]']])







   @slow
   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   str  				=			self.tokenizer_class.from_pretrained('google/electra-base-discriminator')

      __snake_case					:   Optional[Any]  				=			tokenizer.encode('sequence builders'       , add_special_tokens=__a)
      __snake_case					:   List[Any]  				=			tokenizer.encode('multi-sequence build'       , add_special_tokens=__a)

      __snake_case					:   Tuple  				=			tokenizer.build_inputs_with_special_tokens(__a)
      __snake_case					:   int  				=			tokenizer.build_inputs_with_special_tokens(__a       , __a)

      assert encoded_sentence == [1_0_1] + text + [1_0_2]
      assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Optional[int]:
      """simple docstring"""






      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
            __snake_case					:   str  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[Any]  				=			F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
            __snake_case					:   str  				=			tokenizer_r.encode_plus(
                __a       , return_attention_mask=__a       , return_token_type_ids=__a       , return_offsets_mapping=__a       , add_special_tokens=__a       , )

            __snake_case					:   int  				=			tokenizer_r.do_lower_case if hasattr(__a       , 'do_lower_case') else False
            __snake_case					:   Any  				=			(
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'A'),
                    ((1, 2), ','),
                    ((3, 5), 'na'),
                    ((5, 6), '##ï'),
                    ((6, 8), '##ve'),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), 'Allen'),
                    ((2_1, 2_3), '##NL'),
                    ((2_3, 2_4), '##P'),
                    ((2_5, 3_3), 'sentence'),
                    ((3_3, 3_4), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'a'),
                    ((1, 2), ','),
                    ((3, 8), 'naive'),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), 'allen'),
                    ((2_1, 2_3), '##nl'),
                    ((2_3, 2_4), '##p'),
                    ((2_5, 3_3), 'sentence'),
                    ((3_3, 3_4), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )

            self.assertEqual(
                [e[1] for e in expected_results]       , tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
            self.assertEqual([e[0] for e in expected_results]       , tokens['offset_mapping'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Any:
      """simple docstring"""






      __snake_case					:   str  				=			['的', '人', '有']
      __snake_case					:   str  				=			''.join(__a)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
            __snake_case					:   Tuple  				=			True
            __snake_case					:   str  				=			self.tokenizer_class.from_pretrained(__a       , **__a)
            __snake_case					:   Optional[int]  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[str]  				=			tokenizer_p.encode(__a       , add_special_tokens=__a)
            __snake_case					:   Optional[Any]  				=			tokenizer_r.encode(__a       , add_special_tokens=__a)

            __snake_case					:   Union[str, Any]  				=			tokenizer_r.convert_ids_to_tokens(__a)
            __snake_case					:   List[Any]  				=			tokenizer_p.convert_ids_to_tokens(__a)

            # it is expected that each Chinese character is not preceded by "##"
            self.assertListEqual(__a       , __a)
            self.assertListEqual(__a       , __a)

            __snake_case					:   Optional[int]  				=			False
            __snake_case					:   Any  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)
            __snake_case					:   List[str]  				=			self.tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[str]  				=			tokenizer_r.encode(__a       , add_special_tokens=__a)
            __snake_case					:   Optional[Any]  				=			tokenizer_p.encode(__a       , add_special_tokens=__a)

            __snake_case					:   List[str]  				=			tokenizer_r.convert_ids_to_tokens(__a)
            __snake_case					:   Optional[int]  				=			tokenizer_p.convert_ids_to_tokens(__a)

            # it is expected that only the first Chinese character is not preceded by "##".
            __snake_case					:   Any  				=			[
                F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a)
            ]
            self.assertListEqual(__a       , __a)
            self.assertListEqual(__a       , __a)