code: 
  
  
  
  
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  @require_tokenizers
  class      A    (__lowerCamelCase       ,unittest.TestCase				):
        _SCREAMING_SNAKE_CASE					=							RealmTokenizer
        _SCREAMING_SNAKE_CASE					=							RealmTokenizerFast
        _SCREAMING_SNAKE_CASE					=							True
        _SCREAMING_SNAKE_CASE					=							True
        _SCREAMING_SNAKE_CASE					=							filter_non_english
  
  
  
        def    __a (					self					)      ->   Any:
  
  
  
               '''simple docstring'''
  
               super().setUp()
  
               _snake_case      :							Dict     				=						[
                   '''[UNK]''',
                   '''[CLS]''',
                   '''[SEP]''',
                   '''[PAD]''',
                   '''[MASK]''',
                   '''want''',
                   '''##want''',
                   '''##ed''',
                   '''wa''',
                   '''un''',
                   '''runn''',
                   '''##ing''',
                   ''',''',
                   '''low''',
                   '''lowest''',
               ]
               _snake_case      :							Union[str, Any]     				=						os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES['''vocab_file''']					)
               with open(self.vocab_file     ,  '''w'''     ,  encoding='''utf-8'''					) as vocab_writer:
                      vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
  
  
  
        def    __a (					self     ,  lowercase_					)      ->   Dict:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							str     				=						'''UNwant\u00E9d,running'''
               _snake_case      :							List[str]     				=						'''unwanted, running'''
               return input_text, output_text
  
  
  
        def    __a (					self					)      ->   str:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							Any     				=						self.tokenizer_class(self.vocab_file					)
  
               _snake_case      :							int     				=						tokenizer.tokenize('''UNwant\u00E9d,running'''					)
               self.assertListEqual(lowercase_     ,  ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
               self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_					)     ,  [9, 6, 7, 12, 10, 11]					)
  
  
  
        def    __a (					self					)      ->   int:
  
  
  
               '''simple docstring'''
  
               if not self.test_rust_tokenizer:
                      return
  
               _snake_case      :							Any     				=						self.get_tokenizer()
               _snake_case      :							Tuple     				=						self.get_rust_tokenizer()
  
               _snake_case      :							int     				=						'''UNwant\u00E9d,running'''
  
               _snake_case      :							Union[str, Any]     				=						tokenizer.tokenize(lowercase_					)
               _snake_case      :							Optional[int]     				=						rust_tokenizer.tokenize(lowercase_					)
               self.assertListEqual(lowercase_     ,  lowercase_					)
  
               _snake_case      :							Tuple     				=						tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
               _snake_case      :							str     				=						rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
               self.assertListEqual(lowercase_     ,  lowercase_					)
  
               _snake_case      :							str     				=						self.get_rust_tokenizer()
               _snake_case      :							Optional[int]     				=						tokenizer.encode(lowercase_					)
               _snake_case      :							Any     				=						rust_tokenizer.encode(lowercase_					)
               self.assertListEqual(lowercase_     ,  lowercase_					)
  
               # With lower casing
               _snake_case      :							int     				=						self.get_tokenizer(do_lower_case=lowercase_					)
               _snake_case      :							Any     				=						self.get_rust_tokenizer(do_lower_case=lowercase_					)
  
               _snake_case      :							Tuple     				=						'''UNwant\u00E9d,running'''
  
               _snake_case      :							List[str]     				=						tokenizer.tokenize(lowercase_					)
               _snake_case      :							int     				=						rust_tokenizer.tokenize(lowercase_					)
               self.assertListEqual(lowercase_     ,  lowercase_					)
  
               _snake_case      :							int     				=						tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
               _snake_case      :							Dict     				=						rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
               self.assertListEqual(lowercase_     ,  lowercase_					)
  
               _snake_case      :							Optional[Any]     				=						self.get_rust_tokenizer()
               _snake_case      :							int     				=						tokenizer.encode(lowercase_					)
               _snake_case      :							int     				=						rust_tokenizer.encode(lowercase_					)
               self.assertListEqual(lowercase_     ,  lowercase_					)
  
  
  
        def    __a (					self					)      ->   str:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							Optional[Any]     				=						BasicTokenizer()
  
               self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)     ,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)
  
  
  
        def    __a (					self					)      ->   Any:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							Tuple     				=						BasicTokenizer(do_lower_case=lowercase_					)
  
               self.assertListEqual(
                   tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)     ,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
               self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)     ,  ['''hello''']					)
  
  
  
        def    __a (					self					)      ->   Union[str, Any]:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							List[str]     				=						BasicTokenizer(do_lower_case=lowercase_     ,  strip_accents=lowercase_					)
  
               self.assertListEqual(
                   tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)     ,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
               self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)     ,  ['''h\u00E9llo''']					)
  
  
  
        def    __a (					self					)      ->   int:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							Optional[Any]     				=						BasicTokenizer(do_lower_case=lowercase_     ,  strip_accents=lowercase_					)
  
               self.assertListEqual(
                   tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)     ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
               self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)     ,  ['''hello''']					)
  
  
  
        def    __a (					self					)      ->   List[str]:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							List[Any]     				=						BasicTokenizer(do_lower_case=lowercase_					)
  
               self.assertListEqual(
                   tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)     ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
               self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)     ,  ['''hello''']					)
  
  
  
        def    __a (					self					)      ->   str:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							str     				=						BasicTokenizer(do_lower_case=lowercase_					)
  
               self.assertListEqual(
                   tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)     ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
  
  
  
        def    __a (					self					)      ->   Optional[Any]:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							Union[str, Any]     				=						BasicTokenizer(do_lower_case=lowercase_     ,  strip_accents=lowercase_					)
  
               self.assertListEqual(
                   tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)     ,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
  
  
  
        def    __a (					self					)      ->   List[Any]:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							Dict     				=						BasicTokenizer(do_lower_case=lowercase_     ,  strip_accents=lowercase_					)
  
               self.assertListEqual(
                   tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)     ,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
  
  
  
        def    __a (					self					)      ->   Optional[int]:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							str     				=						BasicTokenizer(do_lower_case=lowercase_     ,  never_split=['''[UNK]''']					)
  
               self.assertListEqual(
                   tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)     ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)
  
  
  
        def    __a (					self					)      ->   List[Any]:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							Tuple     				=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']
  
               _snake_case      :							int     				=						{}
               for i, token in enumerate(lowercase_					):
                      _snake_case      :							List[Any]     				=						i
               _snake_case      :							Union[str, Any]     				=						WordpieceTokenizer(vocab=lowercase_     ,  unk_token='''[UNK]'''					)
  
               self.assertListEqual(tokenizer.tokenize(''''''					)     ,  []					)
  
               self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)     ,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)
  
               self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)     ,  ['''[UNK]''', '''runn''', '''##ing''']					)
  
  
  
        def    __a (					self					)      ->   Dict:
  
  
  
               '''simple docstring'''
  
               self.assertTrue(_is_whitespace(''' '''					)					)
               self.assertTrue(_is_whitespace('''\t'''					)					)
               self.assertTrue(_is_whitespace('''\r'''					)					)
               self.assertTrue(_is_whitespace('''\n'''					)					)
               self.assertTrue(_is_whitespace('''\u00A0'''					)					)
  
               self.assertFalse(_is_whitespace('''A'''					)					)
               self.assertFalse(_is_whitespace('''-'''					)					)
  
  
  
        def    __a (					self					)      ->   int:
  
  
  
               '''simple docstring'''
  
               self.assertTrue(_is_control('''\u0005'''					)					)
  
               self.assertFalse(_is_control('''A'''					)					)
               self.assertFalse(_is_control(''' '''					)					)
               self.assertFalse(_is_control('''\t'''					)					)
               self.assertFalse(_is_control('''\r'''					)					)
  
  
  
        def    __a (					self					)      ->   Dict:
  
  
  
               '''simple docstring'''
  
               self.assertTrue(_is_punctuation('''-'''					)					)
               self.assertTrue(_is_punctuation('''$'''					)					)
               self.assertTrue(_is_punctuation('''`'''					)					)
               self.assertTrue(_is_punctuation('''.'''					)					)
  
               self.assertFalse(_is_punctuation('''A'''					)					)
               self.assertFalse(_is_punctuation(''' '''					)					)
  
  
  
        def    __a (					self					)      ->   Any:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							str     				=						self.get_tokenizer()
  
               # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
               self.assertListEqual([tokenizer.tokenize(lowercase_					) for t in ['''Test''', '''\xad''', '''test''']]     ,  [['''[UNK]'''], [], ['''[UNK]''']]					)
  
               if self.test_rust_tokenizer:
                      _snake_case      :							Optional[Any]     				=						self.get_rust_tokenizer()
                      self.assertListEqual(
                          [rust_tokenizer.tokenize(lowercase_					) for t in ['''Test''', '''\xad''', '''test''']]     ,  [['''[UNK]'''], [], ['''[UNK]''']]					)
  
  
  
        @slow
        def    __a (					self					)      ->   Optional[int]:
  
  
  
               '''simple docstring'''
  
               _snake_case      :							Dict     				=						self.tokenizer_class.from_pretrained('''bert-base-uncased'''					)
  
               _snake_case      :							Optional[int]     				=						tokenizer.encode('''sequence builders'''     ,  add_special_tokens=lowercase_					)
               _snake_case      :							List[Any]     				=						tokenizer.encode('''multi-sequence build'''     ,  add_special_tokens=lowercase_					)
  
               _snake_case      :							List[str]     				=						tokenizer.build_inputs_with_special_tokens(lowercase_					)
               _snake_case      :							List[Any]     				=						tokenizer.build_inputs_with_special_tokens(lowercase_     ,  lowercase_					)
  
               assert encoded_sentence == [101] + text + [102]
               assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
        def    __a (					self					)      ->   Optional[int]:
  
  
  
               '''simple docstring'''
  
               for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                             _snake_case      :							Optional[Any]     				=						self.rust_tokenizer_class.from_pretrained(lowercase_     ,  **lowercase_					)
  
                             _snake_case      :							List[str]     				=						F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                             _snake_case      :							Any     				=						tokenizer_r.encode_plus(
                                 lowercase_     ,  return_attention_mask=lowercase_     ,  return_token_type_ids=lowercase_     ,  return_offsets_mapping=lowercase_     ,  add_special_tokens=lowercase_     ,  )
  
                             _snake_case      :							Optional[Any]     				=						tokenizer_r.do_lower_case if hasattr(lowercase_     ,  '''do_lower_case'''					) else False
                             _snake_case      :							str     				=						(
                                 [
                                     ((0, 0), tokenizer_r.cls_token),
                                     ((0, 1), '''A'''),
                                     ((1, 2), ''','''),
                                     ((3, 5), '''na'''),
                                     ((5, 6), '''##ï'''),
                                     ((6, 8), '''##ve'''),
                                     ((9, 15), tokenizer_r.mask_token),
                                     ((16, 21), '''Allen'''),
                                     ((21, 23), '''##NL'''),
                                     ((23, 24), '''##P'''),
                                     ((25, 33), '''sentence'''),
                                     ((33, 34), '''.'''),
                                     ((0, 0), tokenizer_r.sep_token),
                                 ]
                                 if not do_lower_case
                                 else [
                                     ((0, 0), tokenizer_r.cls_token),
                                     ((0, 1), '''a'''),
                                     ((1, 2), ''','''),
                                     ((3, 8), '''naive'''),
                                     ((9, 15), tokenizer_r.mask_token),
                                     ((16, 21), '''allen'''),
                                     ((21, 23), '''##nl'''),
                                     ((23, 24), '''##p'''),
                                     ((25, 33), '''sentence'''),
                                     ((33, 34), '''.'''),
                                     ((0, 0), tokenizer_r.sep_token),
                                 ]
                             )
  
                             self.assertEqual(
                                 [e[1] for e in expected_results]     ,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
                             self.assertEqual([e[0] for e in expected_results]     ,  tokens['''offset_mapping''']					)
  
  
  
        @slow
        def    __a (					self					)      ->   Optional[int]:
  
  
  
               '''simple docstring'''
  
               for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                             _snake_case      :							List[Any]     				=						self.rust_tokenizer_class.from_pretrained(lowercase_     ,  **lowercase_					)
                             _snake_case      :							Optional[int]     				=						self.tokenizer_class.from_pretrained(lowercase_     ,  **lowercase_					)
                             _snake_case      :							Optional[int]     				=						[['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]
  
                             _snake_case      :							Tuple     				=						tokenizer_r.batch_encode_candidates(lowercase_     ,  max_length=10     ,  return_tensors='''np'''					)
                             _snake_case      :							List[Any]     				=						tokenizer_p.batch_encode_candidates(lowercase_     ,  max_length=10     ,  return_tensors='''np'''					)
  
                             _snake_case      :							Optional[Any]     				=						(2, 2, 10)
  
                             self.assertEqual(encoded_sentence_r['''input_ids'''].shape     ,  lowercase_					)
                             self.assertEqual(encoded_sentence_r['''attention_mask'''].shape     ,  lowercase_					)
                             self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape     ,  lowercase_					)
  
                             self.assertEqual(encoded_sentence_p['''input_ids'''].shape     ,  lowercase_					)
                             self.assertEqual(encoded_sentence_p['''attention_mask'''].shape     ,  lowercase_					)
                             self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape     ,  lowercase_					)
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  import numpy as np
  
  
  
  
  def 						A_			(  lowercase_  )			->						Optional[Any]:
         _snake_case      :							Union[str, Any]     				=						np.exp(lowercase_  )
  
         # Add up the all the exponentials
         _snake_case      :							List[str]     				=						np.sum(lowercase_  )
  
         # Divide every exponent by the sum of all exponents
         _snake_case      :							Optional[int]     				=						exponent_vector / sum_of_exponents
  
         return softmax_vector
  
  
  if __name__ == "__main__":
     print(softmax((0,)))
  
  
  
  
  
  
  