code: 
  
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  @require_tokenizers
  class 			a      (      snake_case__      ,unittest.TestCase   ):
  		__lowercase													=		RealmTokenizer
  		__lowercase													=		RealmTokenizerFast
  		__lowercase													=		True
  		__lowercase													=		True
  		__lowercase													=		filter_non_english
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Any:
  						'''simple docstring'''
  
  
  						super().setUp()
  
  						A__     :							Optional[Any]   				=[
  						    '[UNK]',
  						    '[CLS]',
  						    '[SEP]',
  						    '[PAD]',
  						    '[MASK]',
  						    'want',
  						    '##want',
  						    '##ed',
  						    'wa',
  						    'un',
  						    'runn',
  						    '##ing',
  						    ',',
  						    'low',
  						    'lowest',
  						]
  						A__     :							List[Any]   				=os.path.join(self.tmpdirname	,     VOCAB_FILES_NAMES['''vocab_file''']							)
  						with open(self.vocab_file	,     '''w'''	,     encoding='''utf-8'''							) as vocab_writer:
  										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self	,     __UpperCamelCase							)->			Any:
  						'''simple docstring'''
  
  
  						A__     :							Optional[int]   				='UNwant\u00E9d,running'
  						A__     :							List[str]   				='unwanted, running'
  						return input_text, output_text
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Optional[int]:
  						'''simple docstring'''
  
  
  						A__     :							Optional[Any]   				=self.tokenizer_class(self.vocab_file							)
  
  						A__     :							Dict   				=tokenizer.tokenize('''UNwant\u00E9d,running'''							)
  						self.assertListEqual(__UpperCamelCase	,     ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
  						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase							)	,     [9, 6, 7, 12, 10, 11]							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			int:
  						'''simple docstring'''
  
  
  						if not self.test_rust_tokenizer:
  										return
  
  						A__     :							int   				=self.get_tokenizer()
  						A__     :							Optional[Any]   				=self.get_rust_tokenizer()
  
  						A__     :							Optional[Any]   				='UNwant\u00E9d,running'
  
  						A__     :							Optional[int]   				=tokenizer.tokenize(__UpperCamelCase							)
  						A__     :							Optional[int]   				=rust_tokenizer.tokenize(__UpperCamelCase							)
  						self.assertListEqual(__UpperCamelCase	,     __UpperCamelCase							)
  
  						A__     :							Dict   				=tokenizer.encode(__UpperCamelCase	,     add_special_tokens=__UpperCamelCase							)
  						A__     :							List[Any]   				=rust_tokenizer.encode(__UpperCamelCase	,     add_special_tokens=__UpperCamelCase							)
  						self.assertListEqual(__UpperCamelCase	,     __UpperCamelCase							)
  
  						A__     :							Tuple   				=self.get_rust_tokenizer()
  						A__     :							Optional[int]   				=tokenizer.encode(__UpperCamelCase							)
  						A__     :							Tuple   				=rust_tokenizer.encode(__UpperCamelCase							)
  						self.assertListEqual(__UpperCamelCase	,     __UpperCamelCase							)
  
  						# With lower casing
  						A__     :							List[str]   				=self.get_tokenizer(do_lower_case=__UpperCamelCase							)
  						A__     :							List[str]   				=self.get_rust_tokenizer(do_lower_case=__UpperCamelCase							)
  
  						A__     :							str   				='UNwant\u00E9d,running'
  
  						A__     :							int   				=tokenizer.tokenize(__UpperCamelCase							)
  						A__     :							int   				=rust_tokenizer.tokenize(__UpperCamelCase							)
  						self.assertListEqual(__UpperCamelCase	,     __UpperCamelCase							)
  
  						A__     :							Tuple   				=tokenizer.encode(__UpperCamelCase	,     add_special_tokens=__UpperCamelCase							)
  						A__     :							Tuple   				=rust_tokenizer.encode(__UpperCamelCase	,     add_special_tokens=__UpperCamelCase							)
  						self.assertListEqual(__UpperCamelCase	,     __UpperCamelCase							)
  
  						A__     :							Optional[int]   				=self.get_rust_tokenizer()
  						A__     :							Union[str, Any]   				=tokenizer.encode(__UpperCamelCase							)
  						A__     :							Tuple   				=rust_tokenizer.encode(__UpperCamelCase							)
  						self.assertListEqual(__UpperCamelCase	,     __UpperCamelCase							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			List[str]:
  						'''simple docstring'''
  
  
  						A__     :							Tuple   				=BasicTokenizer()
  
  						self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)	,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Optional[int]:
  						'''simple docstring'''
  
  
  						A__     :							str   				=BasicTokenizer(do_lower_case=__UpperCamelCase							)
  
  						self.assertListEqual(
  						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)	,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
  						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)	,     ['''hello''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Union[str, Any]:
  						'''simple docstring'''
  
  
  						A__     :							Optional[int]   				=BasicTokenizer(do_lower_case=__UpperCamelCase	,     strip_accents=__UpperCamelCase							)
  
  						self.assertListEqual(
  						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)	,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
  						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)	,     ['''h\u00E9llo''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Optional[int]:
  						'''simple docstring'''
  
  
  						A__     :							str   				=BasicTokenizer(do_lower_case=__UpperCamelCase	,     strip_accents=__UpperCamelCase							)
  
  						self.assertListEqual(
  						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)	,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
  						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)	,     ['''hello''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			List[Any]:
  						'''simple docstring'''
  
  
  						A__     :							List[Any]   				=BasicTokenizer(do_lower_case=__UpperCamelCase							)
  
  						self.assertListEqual(
  						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)	,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
  						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)	,     ['''hello''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Any:
  						'''simple docstring'''
  
  
  						A__     :							Dict   				=BasicTokenizer(do_lower_case=__UpperCamelCase							)
  
  						self.assertListEqual(
  						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)	,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Union[str, Any]:
  						'''simple docstring'''
  
  
  						A__     :							List[str]   				=BasicTokenizer(do_lower_case=__UpperCamelCase	,     strip_accents=__UpperCamelCase							)
  
  						self.assertListEqual(
  						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)	,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Union[str, Any]:
  						'''simple docstring'''
  
  
  						A__     :							Optional[Any]   				=BasicTokenizer(do_lower_case=__UpperCamelCase	,     strip_accents=__UpperCamelCase							)
  
  						self.assertListEqual(
  						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)	,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			List[Any]:
  						'''simple docstring'''
  
  
  						A__     :							List[Any]   				=BasicTokenizer(do_lower_case=__UpperCamelCase	,     never_split=['''[UNK]''']							)
  
  						self.assertListEqual(
  						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)	,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Union[str, Any]:
  						'''simple docstring'''
  
  
  						A__     :							Optional[Any]   				=['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']
  
  						A__     :							Union[str, Any]   				={}
  						for i, token in enumerate(__UpperCamelCase							):
  										A__     :							List[str]   				=i
  						A__     :							Optional[int]   				=WordpieceTokenizer(vocab=__UpperCamelCase	,     unk_token='''[UNK]'''							)
  
  						self.assertListEqual(tokenizer.tokenize(''''''							)	,     []							)
  
  						self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)	,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)
  
  						self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)	,     ['''[UNK]''', '''runn''', '''##ing''']							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Dict:
  						'''simple docstring'''
  
  
  						self.assertTrue(_is_whitespace(''' '''							)							)
  						self.assertTrue(_is_whitespace('''\t'''							)							)
  						self.assertTrue(_is_whitespace('''\r'''							)							)
  						self.assertTrue(_is_whitespace('''\n'''							)							)
  						self.assertTrue(_is_whitespace('''\u00A0'''							)							)
  
  						self.assertFalse(_is_whitespace('''A'''							)							)
  						self.assertFalse(_is_whitespace('''-'''							)							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Optional[Any]:
  						'''simple docstring'''
  
  
  						self.assertTrue(_is_control('''\u0005'''							)							)
  
  						self.assertFalse(_is_control('''A'''							)							)
  						self.assertFalse(_is_control(''' '''							)							)
  						self.assertFalse(_is_control('''\t'''							)							)
  						self.assertFalse(_is_control('''\r'''							)							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			List[str]:
  						'''simple docstring'''
  
  
  						self.assertTrue(_is_punctuation('''-'''							)							)
  						self.assertTrue(_is_punctuation('''$'''							)							)
  						self.assertTrue(_is_punctuation('''`'''							)							)
  						self.assertTrue(_is_punctuation('''.'''							)							)
  
  						self.assertFalse(_is_punctuation('''A'''							)							)
  						self.assertFalse(_is_punctuation(''' '''							)							)
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			Tuple:
  						'''simple docstring'''
  
  
  						A__     :							Tuple   				=self.get_tokenizer()
  
  						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  						self.assertListEqual([tokenizer.tokenize(__UpperCamelCase							) for t in ['''Test''', '''\xad''', '''test''']]	,     [['''[UNK]'''], [], ['''[UNK]''']]							)
  
  						if self.test_rust_tokenizer:
  										A__     :							List[Any]   				=self.get_rust_tokenizer()
  										self.assertListEqual(
  										    [rust_tokenizer.tokenize(__UpperCamelCase							) for t in ['''Test''', '''\xad''', '''test''']]	,     [['''[UNK]'''], [], ['''[UNK]''']]							)
  
  
  
  
  
  
  		@slow
  		def       lowerCAmelCase_  (  self							)->			Dict:
  						'''simple docstring'''
  
  
  						A__     :							List[str]   				=self.tokenizer_class.from_pretrained('''bert-base-uncased'''							)
  
  						A__     :							Optional[Any]   				=tokenizer.encode('''sequence builders'''	,     add_special_tokens=__UpperCamelCase							)
  						A__     :							Optional[Any]   				=tokenizer.encode('''multi-sequence build'''	,     add_special_tokens=__UpperCamelCase							)
  
  						A__     :							str   				=tokenizer.build_inputs_with_special_tokens(__UpperCamelCase							)
  						A__     :							str   				=tokenizer.build_inputs_with_special_tokens(__UpperCamelCase	,     __UpperCamelCase							)
  
  						assert encoded_sentence == [1_01] + text + [1_02]
  						assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
  
  
  
  
  
  
  		def       lowerCAmelCase_  (  self							)->			str:
  						'''simple docstring'''
  
  
  						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  										with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
  														A__     :							List[Any]   				=self.rust_tokenizer_class.from_pretrained(__UpperCamelCase	,     **__UpperCamelCase							)
  
  														A__     :							int   				=F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
  														A__     :							str   				=tokenizer_r.encode_plus(
  														    __UpperCamelCase	,     return_attention_mask=__UpperCamelCase	,     return_token_type_ids=__UpperCamelCase	,     return_offsets_mapping=__UpperCamelCase	,     add_special_tokens=__UpperCamelCase	,     )
  
  														A__     :							Union[str, Any]   				=tokenizer_r.do_lower_case if hasattr(__UpperCamelCase	,     '''do_lower_case'''							) else False
  														A__     :							Optional[int]   				=(
  														    [
  														        ((0, 0), tokenizer_r.cls_token),
  														        ((0, 1), 'A'),
  														        ((1, 2), ','),
  														        ((3, 5), 'na'),
  														        ((5, 6), '##ï'),
  														        ((6, 8), '##ve'),
  														        ((9, 15), tokenizer_r.mask_token),
  														        ((16, 21), 'Allen'),
  														        ((21, 23), '##NL'),
  														        ((23, 24), '##P'),
  														        ((25, 33), 'sentence'),
  														        ((33, 34), '.'),
  														        ((0, 0), tokenizer_r.sep_token),
  														    ]
  														    if not do_lower_case
  														    else [
  														        ((0, 0), tokenizer_r.cls_token),
  														        ((0, 1), 'a'),
  														        ((1, 2), ','),
  														        ((3, 8), 'naive'),
  														        ((9, 15), tokenizer_r.mask_token),
  														        ((16, 21), 'allen'),
  														        ((21, 23), '##nl'),
  														        ((23, 24), '##p'),
  														        ((25, 33), 'sentence'),
  														        ((33, 34), '.'),
  														        ((0, 0), tokenizer_r.sep_token),
  														    ]
  														)
  
  														self.assertEqual(
  														    [e[1] for e in expected_results]	,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
  														self.assertEqual([e[0] for e in expected_results]	,     tokens['''offset_mapping''']							)
  
  
  
  
  
  
  		@slow
  		def       lowerCAmelCase_  (  self							)->			str:
  						'''simple docstring'''
  
  
  						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  										with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
  														A__     :							List[str]   				=self.rust_tokenizer_class.from_pretrained(__UpperCamelCase	,     **__UpperCamelCase							)
  														A__     :							Dict   				=self.tokenizer_class.from_pretrained(__UpperCamelCase	,     **__UpperCamelCase							)
  														A__     :							List[Any]   				=[['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]
  
  														A__     :							Optional[Any]   				=tokenizer_r.batch_encode_candidates(__UpperCamelCase	,     max_length=10	,     return_tensors='''np'''							)
  														A__     :							int   				=tokenizer_p.batch_encode_candidates(__UpperCamelCase	,     max_length=10	,     return_tensors='''np'''							)
  
  														A__     :							str   				=(2, 2, 10)
  
  														self.assertEqual(encoded_sentence_r['''input_ids'''].shape	,     __UpperCamelCase							)
  														self.assertEqual(encoded_sentence_r['''attention_mask'''].shape	,     __UpperCamelCase							)
  														self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape	,     __UpperCamelCase							)
  
  														self.assertEqual(encoded_sentence_p['''input_ids'''].shape	,     __UpperCamelCase							)
  														self.assertEqual(encoded_sentence_p['''attention_mask'''].shape	,     __UpperCamelCase							)
  														self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape	,     __UpperCamelCase							)
  
  
  
  
style_context: 
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_	)			->							int:
  				if number != int(snake_case_	):
  								raise ValueError('''factorial() only accepts integral values'''	)
  				if number < 0:
  								raise ValueError('''factorial() not defined for negative values'''	)
  				A__     :							Union[str, Any]   				=1
  				for i in range(1,	number + 1	):
  								value *= i
  				return value
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_	)			->							int:
  				if not isinstance(snake_case_,	snake_case_	):
  								raise ValueError('''factorial() only accepts integral values'''	)
  				if n < 0:
  								raise ValueError('''factorial() not defined for negative values'''	)
  				return 1 if n == 0 or n == 1 else n * factorial(n - 1	)
  
  
  if __name__ == "__main__":
  						import doctest
  
  						doctest.testmod()
  
  						__lowerCamelCase       :       Any				   =			int(input("Enter a positive integer: ").strip() or 0)
  						print(F"factorial{n} is {factorial(n)}")
  
  
  
  