code: 
  
  
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  @require_tokenizers
  class 		_snake_case      (			lowerCamelCase					,unittest.TestCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			RealmTokenizer
   lowerCamelCase_          =			RealmTokenizerFast
   lowerCamelCase_          =			True
   lowerCamelCase_          =			True
   lowerCamelCase_          =			filter_non_english
   def lowercase_		(						self					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     super().setUp()
  
     _A           =    [
         '''[UNK]''',
         '''[CLS]''',
         '''[SEP]''',
         '''[PAD]''',
         '''[MASK]''',
         '''want''',
         '''##want''',
         '''##ed''',
         '''wa''',
         '''un''',
         '''runn''',
         '''##ing''',
         ''',''',
         '''low''',
         '''lowest''',
     ]
     _A           =    os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES['''vocab_file''']					)
     with open(self.vocab_file	,    '''w'''	,    encoding='''utf-8'''					) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
   def lowercase_		(						self	,    a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    '''UNwant\u00E9d,running'''
     _A           =    '''unwanted, running'''
     return input_text, output_text
   def lowercase_		(						self					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.tokenizer_class(self.vocab_file					)
  
     _A           =    tokenizer.tokenize('''UNwant\u00E9d,running'''					)
     self.assertListEqual(a	,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(a					)	,    [9, 6, 7, 1_2, 1_0, 1_1]					)
   def lowercase_		(						self					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     if not self.test_rust_tokenizer:
       return
  
     _A           =    self.get_tokenizer()
     _A           =    self.get_rust_tokenizer()
  
     _A           =    '''UNwant\u00E9d,running'''
  
     _A           =    tokenizer.tokenize(a					)
     _A           =    rust_tokenizer.tokenize(a					)
     self.assertListEqual(a	,    a					)
  
     _A           =    tokenizer.encode(a	,    add_special_tokens=a					)
     _A           =    rust_tokenizer.encode(a	,    add_special_tokens=a					)
     self.assertListEqual(a	,    a					)
  
     _A           =    self.get_rust_tokenizer()
     _A           =    tokenizer.encode(a					)
     _A           =    rust_tokenizer.encode(a					)
     self.assertListEqual(a	,    a					)
  
     # With lower casing
     _A           =    self.get_tokenizer(do_lower_case=a					)
     _A           =    self.get_rust_tokenizer(do_lower_case=a					)
  
     _A           =    '''UNwant\u00E9d,running'''
  
     _A           =    tokenizer.tokenize(a					)
     _A           =    rust_tokenizer.tokenize(a					)
     self.assertListEqual(a	,    a					)
  
     _A           =    tokenizer.encode(a	,    add_special_tokens=a					)
     _A           =    rust_tokenizer.encode(a	,    add_special_tokens=a					)
     self.assertListEqual(a	,    a					)
  
     _A           =    self.get_rust_tokenizer()
     _A           =    tokenizer.encode(a					)
     _A           =    rust_tokenizer.encode(a					)
     self.assertListEqual(a	,    a					)
   def lowercase_		(						self					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer()
  
     self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)	,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)
   def lowercase_		(						self					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=a					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)	,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
   def lowercase_		(						self					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=a	,    strip_accents=a					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''h\u00E9llo''']					)
   def lowercase_		(						self					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=a	,    strip_accents=a					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
   def lowercase_		(						self					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=a					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
   def lowercase_		(						self					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=a					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)	,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
   def lowercase_		(						self					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=a	,    strip_accents=a					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
   def lowercase_		(						self					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=a	,    strip_accents=a					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
   def lowercase_		(						self					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=a	,    never_split=['''[UNK]''']					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)	,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)
   def lowercase_		(						self					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']
  
     _A           =    {}
     for i, token in enumerate(a					):
       _A           =    i
     _A           =    WordpieceTokenizer(vocab=a	,    unk_token='''[UNK]'''					)
  
     self.assertListEqual(tokenizer.tokenize(''''''					)	,    []					)
  
     self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)	,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)
  
     self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)	,    ['''[UNK]''', '''runn''', '''##ing''']					)
   def lowercase_		(						self					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     self.assertTrue(_is_whitespace(''' '''					)					)
     self.assertTrue(_is_whitespace('''\t'''					)					)
     self.assertTrue(_is_whitespace('''\r'''					)					)
     self.assertTrue(_is_whitespace('''\n'''					)					)
     self.assertTrue(_is_whitespace('''\u00A0'''					)					)
  
     self.assertFalse(_is_whitespace('''A'''					)					)
     self.assertFalse(_is_whitespace('''-'''					)					)
   def lowercase_		(						self					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     self.assertTrue(_is_control('''\u0005'''					)					)
  
     self.assertFalse(_is_control('''A'''					)					)
     self.assertFalse(_is_control(''' '''					)					)
     self.assertFalse(_is_control('''\t'''					)					)
     self.assertFalse(_is_control('''\r'''					)					)
   def lowercase_		(						self					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     self.assertTrue(_is_punctuation('''-'''					)					)
     self.assertTrue(_is_punctuation('''$'''					)					)
     self.assertTrue(_is_punctuation('''`'''					)					)
     self.assertTrue(_is_punctuation('''.'''					)					)
  
     self.assertFalse(_is_punctuation('''A'''					)					)
     self.assertFalse(_is_punctuation(''' '''					)					)
   def lowercase_		(						self					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.get_tokenizer()
  
     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(a					) for t in ['''Test''', '''\xad''', '''test''']]	,    [['''[UNK]'''], [], ['''[UNK]''']]					)
  
     if self.test_rust_tokenizer:
       _A           =    self.get_rust_tokenizer()
       self.assertListEqual(
           [rust_tokenizer.tokenize(a					) for t in ['''Test''', '''\xad''', '''test''']]	,    [['''[UNK]'''], [], ['''[UNK]''']]					)
   @slow
   def lowercase_		(						self					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.tokenizer_class.from_pretrained('''bert-base-uncased'''					)
  
     _A           =    tokenizer.encode('''sequence builders'''	,    add_special_tokens=a					)
     _A           =    tokenizer.encode('''multi-sequence build'''	,    add_special_tokens=a					)
  
     _A           =    tokenizer.build_inputs_with_special_tokens(a					)
     _A           =    tokenizer.build_inputs_with_special_tokens(a	,    a					)
  
     assert encoded_sentence == [1_0_1] + text + [1_0_2]
     assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
   def lowercase_		(						self					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
         _A           =    self.rust_tokenizer_class.from_pretrained(a	,    **a					)
  
         _A           =    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
         _A           =    tokenizer_r.encode_plus(
             a	,    return_attention_mask=a	,    return_token_type_ids=a	,    return_offsets_mapping=a	,    add_special_tokens=a	,    )
  
         _A           =    tokenizer_r.do_lower_case if hasattr(a	,    '''do_lower_case'''					) else False
         _A           =    (
             [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), '''A'''),
                 ((1, 2), ''','''),
                 ((3, 5), '''na'''),
                 ((5, 6), '''##ï'''),
                 ((6, 8), '''##ve'''),
                 ((9, 1_5), tokenizer_r.mask_token),
                 ((1_6, 2_1), '''Allen'''),
                 ((2_1, 2_3), '''##NL'''),
                 ((2_3, 2_4), '''##P'''),
                 ((2_5, 3_3), '''sentence'''),
                 ((3_3, 3_4), '''.'''),
                 ((0, 0), tokenizer_r.sep_token),
             ]
             if not do_lower_case
             else [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), '''a'''),
                 ((1, 2), ''','''),
                 ((3, 8), '''naive'''),
                 ((9, 1_5), tokenizer_r.mask_token),
                 ((1_6, 2_1), '''allen'''),
                 ((2_1, 2_3), '''##nl'''),
                 ((2_3, 2_4), '''##p'''),
                 ((2_5, 3_3), '''sentence'''),
                 ((3_3, 3_4), '''.'''),
                 ((0, 0), tokenizer_r.sep_token),
             ]
         )
  
         self.assertEqual(
             [e[1] for e in expected_results]	,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
         self.assertEqual([e[0] for e in expected_results]	,    tokens['''offset_mapping''']					)
  
  
  
   @slow
   def lowercase_		(						self					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
         _A           =    self.rust_tokenizer_class.from_pretrained(a	,    **a					)
         _A           =    self.tokenizer_class.from_pretrained(a	,    **a					)
         _A           =    [['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]
  
         _A           =    tokenizer_r.batch_encode_candidates(a	,    max_length=1_0	,    return_tensors='''np'''					)
         _A           =    tokenizer_p.batch_encode_candidates(a	,    max_length=1_0	,    return_tensors='''np'''					)
  
         _A           =    (2, 2, 1_0)
  
         self.assertEqual(encoded_sentence_r['''input_ids'''].shape	,    a					)
         self.assertEqual(encoded_sentence_r['''attention_mask'''].shape	,    a					)
         self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape	,    a					)
  
         self.assertEqual(encoded_sentence_p['''input_ids'''].shape	,    a					)
         self.assertEqual(encoded_sentence_p['''attention_mask'''].shape	,    a					)
         self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape	,    a					)
style_context: 
  
  
  
  import platform
  from argparse import ArgumentParser
  
  import huggingface_hub
  import pandas
  import pyarrow
  
  from datasets import __version__ as version
  from datasets.commands import BaseDatasetsCLICommand
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case							)   -> Optional[Any]:
    return EnvironmentCommand()
  
  
  
  class 		_snake_case      (			lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   @staticmethod
   def lowercase_		(						a					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    parser.add_parser('''env'''	,    help='''Print relevant system environment info.'''					)
     download_parser.set_defaults(func=a					)
   def lowercase_		(						self					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    {
         '''`datasets` version''': version,
         '''Platform''': platform.platform(),
         '''Python version''': platform.python_version(),
         '''Huggingface_hub version''': huggingface_hub.__version__,
         '''PyArrow version''': pyarrow.__version__,
         '''Pandas version''': pandas.__version__,
     }
  
     print('''\nCopy-and-paste the text below in your GitHub issue.\n'''					)
     print(self.format_dict(a					)					)
  
     return info
  
  
  
   @staticmethod
   def lowercase_		(						a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return "\n".join([f'''- {prop}: {val}''' for prop, val in d.items()]					) + "\n"