


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class   __lowerCamelCase (_a							,			unittest.TestCase	):
       _lowercase       							=    RealmTokenizer
       _lowercase       							=    RealmTokenizerFast
       _lowercase       							=    True
       _lowercase       							=    True
       _lowercase       							=    filter_non_english







       def 		snake_case_  ( self:   Union[str, Any]				):






            '''simple docstring'''




            super().setUp()

            __UpperCamelCase   = [
                '[UNK]',
                '[CLS]',
                '[SEP]',
                '[PAD]',
                '[MASK]',
                'want',
                '##want',
                '##ed',
                'wa',
                'un',
                'runn',
                '##ing',
                ',',
                'low',
                'lowest',
            ]
            __UpperCamelCase   = os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['vocab_file']				)
            with open(self.vocab_file,'w',encoding='utf-8'				) as vocab_writer:
                 vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)







       def 		snake_case_  ( self:   Optional[int],A_:   Optional[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = 'UNwant\u00E9d,running'
            __UpperCamelCase   = 'unwanted, running'
            return input_text, output_text







       def 		snake_case_  ( self:   List[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = self.tokenizer_class(self.vocab_file				)

            __UpperCamelCase   = tokenizer.tokenize('UNwant\u00E9d,running'				)
            self.assertListEqual(A_,['un', '##want', '##ed', ',', 'runn', '##ing']				)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(A_				),[9, 6, 7, 12, 10, 11]				)







       def 		snake_case_  ( self:   Tuple				):






            '''simple docstring'''




            if not self.test_rust_tokenizer:
                 return

            __UpperCamelCase   = self.get_tokenizer()
            __UpperCamelCase   = self.get_rust_tokenizer()

            __UpperCamelCase   = 'UNwant\u00E9d,running'

            __UpperCamelCase   = tokenizer.tokenize(A_				)
            __UpperCamelCase   = rust_tokenizer.tokenize(A_				)
            self.assertListEqual(A_,A_				)

            __UpperCamelCase   = tokenizer.encode(A_,add_special_tokens=A_				)
            __UpperCamelCase   = rust_tokenizer.encode(A_,add_special_tokens=A_				)
            self.assertListEqual(A_,A_				)

            __UpperCamelCase   = self.get_rust_tokenizer()
            __UpperCamelCase   = tokenizer.encode(A_				)
            __UpperCamelCase   = rust_tokenizer.encode(A_				)
            self.assertListEqual(A_,A_				)

            # With lower casing
            __UpperCamelCase   = self.get_tokenizer(do_lower_case=A_				)
            __UpperCamelCase   = self.get_rust_tokenizer(do_lower_case=A_				)

            __UpperCamelCase   = 'UNwant\u00E9d,running'

            __UpperCamelCase   = tokenizer.tokenize(A_				)
            __UpperCamelCase   = rust_tokenizer.tokenize(A_				)
            self.assertListEqual(A_,A_				)

            __UpperCamelCase   = tokenizer.encode(A_,add_special_tokens=A_				)
            __UpperCamelCase   = rust_tokenizer.encode(A_,add_special_tokens=A_				)
            self.assertListEqual(A_,A_				)

            __UpperCamelCase   = self.get_rust_tokenizer()
            __UpperCamelCase   = tokenizer.encode(A_				)
            __UpperCamelCase   = rust_tokenizer.encode(A_				)
            self.assertListEqual(A_,A_				)







       def 		snake_case_  ( self:   Dict				):






            '''simple docstring'''




            __UpperCamelCase   = BasicTokenizer()

            self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'				),['ah', '\u535A', '\u63A8', 'zz']				)







       def 		snake_case_  ( self:   Union[str, Any]				):






            '''simple docstring'''




            __UpperCamelCase   = BasicTokenizer(do_lower_case=A_				)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				),['hello', '!', 'how', 'are', 'you', '?']				)
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				),['hello']				)







       def 		snake_case_  ( self:   int				):






            '''simple docstring'''




            __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,strip_accents=A_				)

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['hällo', '!', 'how', 'are', 'you', '?']				)
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				),['h\u00E9llo']				)







       def 		snake_case_  ( self:   Optional[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,strip_accents=A_				)

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['hallo', '!', 'how', 'are', 'you', '?']				)
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				),['hello']				)







       def 		snake_case_  ( self:   List[str]				):






            '''simple docstring'''




            __UpperCamelCase   = BasicTokenizer(do_lower_case=A_				)

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['hallo', '!', 'how', 'are', 'you', '?']				)
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				),['hello']				)







       def 		snake_case_  ( self:   str				):






            '''simple docstring'''




            __UpperCamelCase   = BasicTokenizer(do_lower_case=A_				)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				),['HeLLo', '!', 'how', 'Are', 'yoU', '?']				)







       def 		snake_case_  ( self:   Optional[int]				):






            '''simple docstring'''




            __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,strip_accents=A_				)

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['HäLLo', '!', 'how', 'Are', 'yoU', '?']				)







       def 		snake_case_  ( self:   str				):






            '''simple docstring'''




            __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,strip_accents=A_				)

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['HaLLo', '!', 'how', 'Are', 'yoU', '?']				)







       def 		snake_case_  ( self:   List[str]				):






            '''simple docstring'''




            __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,never_split=['[UNK]']				)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'				),['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']				)







       def 		snake_case_  ( self:   Union[str, Any]				):






            '''simple docstring'''




            __UpperCamelCase   = ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

            __UpperCamelCase   = {}
            for i, token in enumerate(A_				):
                 __UpperCamelCase   = i
            __UpperCamelCase   = WordpieceTokenizer(vocab=A_,unk_token='[UNK]'				)

            self.assertListEqual(tokenizer.tokenize(''				),[]				)

            self.assertListEqual(tokenizer.tokenize('unwanted running'				),['un', '##want', '##ed', 'runn', '##ing']				)

            self.assertListEqual(tokenizer.tokenize('unwantedX running'				),['[UNK]', 'runn', '##ing']				)







       def 		snake_case_  ( self:   Any				):






            '''simple docstring'''




            self.assertTrue(_is_whitespace(' '				)				)
            self.assertTrue(_is_whitespace('\t'				)				)
            self.assertTrue(_is_whitespace('\r'				)				)
            self.assertTrue(_is_whitespace('\n'				)				)
            self.assertTrue(_is_whitespace('\u00A0'				)				)

            self.assertFalse(_is_whitespace('A'				)				)
            self.assertFalse(_is_whitespace('-'				)				)







       def 		snake_case_  ( self:   Optional[int]				):






            '''simple docstring'''




            self.assertTrue(_is_control('\u0005'				)				)

            self.assertFalse(_is_control('A'				)				)
            self.assertFalse(_is_control(' '				)				)
            self.assertFalse(_is_control('\t'				)				)
            self.assertFalse(_is_control('\r'				)				)







       def 		snake_case_  ( self:   str				):






            '''simple docstring'''




            self.assertTrue(_is_punctuation('-'				)				)
            self.assertTrue(_is_punctuation('$'				)				)
            self.assertTrue(_is_punctuation('`'				)				)
            self.assertTrue(_is_punctuation('.'				)				)

            self.assertFalse(_is_punctuation('A'				)				)
            self.assertFalse(_is_punctuation(' '				)				)







       def 		snake_case_  ( self:   List[str]				):






            '''simple docstring'''




            __UpperCamelCase   = self.get_tokenizer()

            # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
            self.assertListEqual([tokenizer.tokenize(A_				) for t in ['Test', '\xad', 'test']],[['[UNK]'], [], ['[UNK]']]				)

            if self.test_rust_tokenizer:
                 __UpperCamelCase   = self.get_rust_tokenizer()
                 self.assertListEqual(
                     [rust_tokenizer.tokenize(A_				) for t in ['Test', '\xad', 'test']],[['[UNK]'], [], ['[UNK]']]				)







       @slow
       def 		snake_case_  ( self:   str				):






            '''simple docstring'''




            __UpperCamelCase   = self.tokenizer_class.from_pretrained('bert-base-uncased'				)

            __UpperCamelCase   = tokenizer.encode('sequence builders',add_special_tokens=A_				)
            __UpperCamelCase   = tokenizer.encode('multi-sequence build',add_special_tokens=A_				)

            __UpperCamelCase   = tokenizer.build_inputs_with_special_tokens(A_				)
            __UpperCamelCase   = tokenizer.build_inputs_with_special_tokens(A_,A_				)

            assert encoded_sentence == [101] + text + [102]
            assert encoded_pair == [101] + text + [102] + text_a + [102]







       def 		snake_case_  ( self:   Tuple				):






            '''simple docstring'''




            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
                      __UpperCamelCase   = self.rust_tokenizer_class.from_pretrained(A_,**A_				)

                      __UpperCamelCase   = F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                      __UpperCamelCase   = tokenizer_r.encode_plus(
                          A_,return_attention_mask=A_,return_token_type_ids=A_,return_offsets_mapping=A_,add_special_tokens=A_,)

                      __UpperCamelCase   = tokenizer_r.do_lower_case if hasattr(A_,'do_lower_case'				) else False
                      __UpperCamelCase   = (
                          [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), 'A'),
                              ((1, 2), ','),
                              ((3, 5), 'na'),
                              ((5, 6), '##ï'),
                              ((6, 8), '##ve'),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), 'Allen'),
                              ((21, 23), '##NL'),
                              ((23, 24), '##P'),
                              ((25, 33), 'sentence'),
                              ((33, 34), '.'),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                          if not do_lower_case
                          else [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), 'a'),
                              ((1, 2), ','),
                              ((3, 8), 'naive'),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), 'allen'),
                              ((21, 23), '##nl'),
                              ((23, 24), '##p'),
                              ((25, 33), 'sentence'),
                              ((33, 34), '.'),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                      )

                      self.assertEqual(
                          [e[1] for e in expected_results],tokenizer_r.convert_ids_to_tokens(tokens['input_ids']				)				)
                      self.assertEqual([e[0] for e in expected_results],tokens['offset_mapping']				)







       @slow
       def 		snake_case_  ( self:   Any				):






            '''simple docstring'''




            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
                      __UpperCamelCase   = self.rust_tokenizer_class.from_pretrained(A_,**A_				)
                      __UpperCamelCase   = self.tokenizer_class.from_pretrained(A_,**A_				)
                      __UpperCamelCase   = [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

                      __UpperCamelCase   = tokenizer_r.batch_encode_candidates(A_,max_length=10,return_tensors='np'				)
                      __UpperCamelCase   = tokenizer_p.batch_encode_candidates(A_,max_length=10,return_tensors='np'				)

                      __UpperCamelCase   = (2, 2, 10)

                      self.assertEqual(encoded_sentence_r['input_ids'].shape,A_				)
                      self.assertEqual(encoded_sentence_r['attention_mask'].shape,A_				)
                      self.assertEqual(encoded_sentence_r['token_type_ids'].shape,A_				)

                      self.assertEqual(encoded_sentence_p['input_ids'].shape,A_				)
                      self.assertEqual(encoded_sentence_p['attention_mask'].shape,A_				)
                      self.assertEqual(encoded_sentence_p['token_type_ids'].shape,A_				)






def 		_A		(		_lowercase   )				->   bool:


     """simple docstring"""


     if not isinstance(_lowercase				,     _lowercase   ):
          raise ValueError('Input series is not valid, valid series - [1, 2/3, 2]'   )
     if len(_lowercase   ) == 0:
          raise ValueError('Input list must be a non empty list'   )
     if len(_lowercase   ) == 1 and series[0] != 0:
          return True
     __UpperCamelCase   = []
     __UpperCamelCase   = len(_lowercase   )
     for i in range(0				,     _lowercase   ):
          if series[i] == 0:
               raise ValueError('Input series cannot have 0 as an element'   )
          rec_series.append(1 / series[i]   )
     __UpperCamelCase   = rec_series[1] - rec_series[0]
     for index in range(2				,     _lowercase   ):
          if rec_series[index] - rec_series[index - 1] != common_diff:
               return False
     return True






def 		_A		(		_lowercase   )				->   float:


     """simple docstring"""


     if not isinstance(_lowercase				,     _lowercase   ):
          raise ValueError('Input series is not valid, valid series - [2, 4, 6]'   )
     if len(_lowercase   ) == 0:
          raise ValueError('Input list must be a non empty list'   )
     __UpperCamelCase   = 0
     for val in series:
          answer += 1 / val
     return len(_lowercase   ) / answer


if __name__ == "__main__":
    import doctest

    doctest.testmod()


