



'''simple docstring'''





import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 		_snake_case      (			lowerCamelCase					,unittest.TestCase				):





 """simple docstring"""



 lowerCamelCase_          =			RealmTokenizer
 lowerCamelCase_          =			RealmTokenizerFast
 lowerCamelCase_          =			True
 lowerCamelCase_          =			True
 lowerCamelCase_          =			filter_non_english
 def lowercase_		(						self					)	->							Any:




   """simple docstring"""







   super().setUp()

   _A           =    [
       '''[UNK]''',
       '''[CLS]''',
       '''[SEP]''',
       '''[PAD]''',
       '''[MASK]''',
       '''want''',
       '''##want''',
       '''##ed''',
       '''wa''',
       '''un''',
       '''runn''',
       '''##ing''',
       ''',''',
       '''low''',
       '''lowest''',
   ]
   _A           =    os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES['''vocab_file''']					)
   with open(self.vocab_file	,    '''w'''	,    encoding='''utf-8'''					) as vocab_writer:
     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
 def lowercase_		(						self	,    a					)	->							Tuple:




   """simple docstring"""







   _A           =    '''UNwant\u00E9d,running'''
   _A           =    '''unwanted, running'''
   return input_text, output_text
 def lowercase_		(						self					)	->							Any:




   """simple docstring"""







   _A           =    self.tokenizer_class(self.vocab_file					)

   _A           =    tokenizer.tokenize('''UNwant\u00E9d,running'''					)
   self.assertListEqual(a	,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
   self.assertListEqual(tokenizer.convert_tokens_to_ids(a					)	,    [9, 6, 7, 1_2, 1_0, 1_1]					)
 def lowercase_		(						self					)	->							Optional[int]:




   """simple docstring"""







   if not self.test_rust_tokenizer:
     return

   _A           =    self.get_tokenizer()
   _A           =    self.get_rust_tokenizer()

   _A           =    '''UNwant\u00E9d,running'''

   _A           =    tokenizer.tokenize(a					)
   _A           =    rust_tokenizer.tokenize(a					)
   self.assertListEqual(a	,    a					)

   _A           =    tokenizer.encode(a	,    add_special_tokens=a					)
   _A           =    rust_tokenizer.encode(a	,    add_special_tokens=a					)
   self.assertListEqual(a	,    a					)

   _A           =    self.get_rust_tokenizer()
   _A           =    tokenizer.encode(a					)
   _A           =    rust_tokenizer.encode(a					)
   self.assertListEqual(a	,    a					)

   # With lower casing
   _A           =    self.get_tokenizer(do_lower_case=a					)
   _A           =    self.get_rust_tokenizer(do_lower_case=a					)

   _A           =    '''UNwant\u00E9d,running'''

   _A           =    tokenizer.tokenize(a					)
   _A           =    rust_tokenizer.tokenize(a					)
   self.assertListEqual(a	,    a					)

   _A           =    tokenizer.encode(a	,    add_special_tokens=a					)
   _A           =    rust_tokenizer.encode(a	,    add_special_tokens=a					)
   self.assertListEqual(a	,    a					)

   _A           =    self.get_rust_tokenizer()
   _A           =    tokenizer.encode(a					)
   _A           =    rust_tokenizer.encode(a					)
   self.assertListEqual(a	,    a					)
 def lowercase_		(						self					)	->							int:




   """simple docstring"""







   _A           =    BasicTokenizer()

   self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)	,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)
 def lowercase_		(						self					)	->							Optional[Any]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=a					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)	,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
 def lowercase_		(						self					)	->							List[str]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=a	,    strip_accents=a					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''h\u00E9llo''']					)
 def lowercase_		(						self					)	->							Dict:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=a	,    strip_accents=a					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
 def lowercase_		(						self					)	->							List[Any]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=a					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
 def lowercase_		(						self					)	->							Optional[int]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=a					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)	,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
 def lowercase_		(						self					)	->							str:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=a	,    strip_accents=a					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
 def lowercase_		(						self					)	->							Tuple:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=a	,    strip_accents=a					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
 def lowercase_		(						self					)	->							Dict:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=a	,    never_split=['''[UNK]''']					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)	,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)
 def lowercase_		(						self					)	->							List[str]:




   """simple docstring"""







   _A           =    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

   _A           =    {}
   for i, token in enumerate(a					):
     _A           =    i
   _A           =    WordpieceTokenizer(vocab=a	,    unk_token='''[UNK]'''					)

   self.assertListEqual(tokenizer.tokenize(''''''					)	,    []					)

   self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)	,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)

   self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)	,    ['''[UNK]''', '''runn''', '''##ing''']					)
 def lowercase_		(						self					)	->							Optional[Any]:




   """simple docstring"""







   self.assertTrue(_is_whitespace(''' '''					)					)
   self.assertTrue(_is_whitespace('''\t'''					)					)
   self.assertTrue(_is_whitespace('''\r'''					)					)
   self.assertTrue(_is_whitespace('''\n'''					)					)
   self.assertTrue(_is_whitespace('''\u00A0'''					)					)

   self.assertFalse(_is_whitespace('''A'''					)					)
   self.assertFalse(_is_whitespace('''-'''					)					)
 def lowercase_		(						self					)	->							Optional[Any]:




   """simple docstring"""







   self.assertTrue(_is_control('''\u0005'''					)					)

   self.assertFalse(_is_control('''A'''					)					)
   self.assertFalse(_is_control(''' '''					)					)
   self.assertFalse(_is_control('''\t'''					)					)
   self.assertFalse(_is_control('''\r'''					)					)
 def lowercase_		(						self					)	->							Dict:




   """simple docstring"""







   self.assertTrue(_is_punctuation('''-'''					)					)
   self.assertTrue(_is_punctuation('''$'''					)					)
   self.assertTrue(_is_punctuation('''`'''					)					)
   self.assertTrue(_is_punctuation('''.'''					)					)

   self.assertFalse(_is_punctuation('''A'''					)					)
   self.assertFalse(_is_punctuation(''' '''					)					)
 def lowercase_		(						self					)	->							Tuple:




   """simple docstring"""







   _A           =    self.get_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(a					) for t in ['''Test''', '''\xad''', '''test''']]	,    [['''[UNK]'''], [], ['''[UNK]''']]					)

   if self.test_rust_tokenizer:
     _A           =    self.get_rust_tokenizer()
     self.assertListEqual(
         [rust_tokenizer.tokenize(a					) for t in ['''Test''', '''\xad''', '''test''']]	,    [['''[UNK]'''], [], ['''[UNK]''']]					)
 @slow
 def lowercase_		(						self					)	->							Dict:




   """simple docstring"""







   _A           =    self.tokenizer_class.from_pretrained('''bert-base-uncased'''					)

   _A           =    tokenizer.encode('''sequence builders'''	,    add_special_tokens=a					)
   _A           =    tokenizer.encode('''multi-sequence build'''	,    add_special_tokens=a					)

   _A           =    tokenizer.build_inputs_with_special_tokens(a					)
   _A           =    tokenizer.build_inputs_with_special_tokens(a	,    a					)

   assert encoded_sentence == [1_0_1] + text + [1_0_2]
   assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
 def lowercase_		(						self					)	->							Optional[Any]:




   """simple docstring"""







   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
       _A           =    self.rust_tokenizer_class.from_pretrained(a	,    **a					)

       _A           =    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
       _A           =    tokenizer_r.encode_plus(
           a	,    return_attention_mask=a	,    return_token_type_ids=a	,    return_offsets_mapping=a	,    add_special_tokens=a	,    )

       _A           =    tokenizer_r.do_lower_case if hasattr(a	,    '''do_lower_case'''					) else False
       _A           =    (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), '''A'''),
               ((1, 2), ''','''),
               ((3, 5), '''na'''),
               ((5, 6), '''##ï'''),
               ((6, 8), '''##ve'''),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), '''Allen'''),
               ((2_1, 2_3), '''##NL'''),
               ((2_3, 2_4), '''##P'''),
               ((2_5, 3_3), '''sentence'''),
               ((3_3, 3_4), '''.'''),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), '''a'''),
               ((1, 2), ''','''),
               ((3, 8), '''naive'''),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), '''allen'''),
               ((2_1, 2_3), '''##nl'''),
               ((2_3, 2_4), '''##p'''),
               ((2_5, 3_3), '''sentence'''),
               ((3_3, 3_4), '''.'''),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]	,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
       self.assertEqual([e[0] for e in expected_results]	,    tokens['''offset_mapping''']					)



 @slow
 def lowercase_		(						self					)	->							Optional[int]:




   """simple docstring"""







   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
       _A           =    self.rust_tokenizer_class.from_pretrained(a	,    **a					)
       _A           =    self.tokenizer_class.from_pretrained(a	,    **a					)
       _A           =    [['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

       _A           =    tokenizer_r.batch_encode_candidates(a	,    max_length=1_0	,    return_tensors='''np'''					)
       _A           =    tokenizer_p.batch_encode_candidates(a	,    max_length=1_0	,    return_tensors='''np'''					)

       _A           =    (2, 2, 1_0)

       self.assertEqual(encoded_sentence_r['''input_ids'''].shape	,    a					)
       self.assertEqual(encoded_sentence_r['''attention_mask'''].shape	,    a					)
       self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape	,    a					)

       self.assertEqual(encoded_sentence_p['''input_ids'''].shape	,    a					)
       self.assertEqual(encoded_sentence_p['''attention_mask'''].shape	,    a					)
       self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape	,    a					)




from ..utils import requires_backends


__lowerCamelCase  				=					None



class 		_snake_case      :





 """simple docstring"""



 def __init__(						self	,    *a	,    **a					)	->							Tuple:




   """simple docstring"""







   requires_backends(self	,    ['''detectron2''']					)



 @classmethod
 def lowercase_		(						cls	,    *a	,    **a					)	->							Optional[int]:




   """simple docstring"""







   requires_backends(cls	,    ['''detectron2''']					)