code: 
  
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  @require_tokenizers
  class 		_SCREAMING_SNAKE_CASE    (	A       ,  unittest.TestCase    ):
       __SCREAMING_SNAKE_CASE    				=						RealmTokenizer
       __SCREAMING_SNAKE_CASE    				=						RealmTokenizerFast
       __SCREAMING_SNAKE_CASE    				=						True
       __SCREAMING_SNAKE_CASE    				=						True
       __SCREAMING_SNAKE_CASE    				=						filter_non_english
  
  
       def 					__snake_case(     self						):
         super().setUp()
  
         _UpperCAmelCase		:       Optional[int]       =    [
             """[UNK]""",
             """[CLS]""",
             """[SEP]""",
             """[PAD]""",
             """[MASK]""",
             """want""",
             """##want""",
             """##ed""",
             """wa""",
             """un""",
             """runn""",
             """##ing""",
             """,""",
             """low""",
             """lowest""",
         ]
         _UpperCAmelCase		:       List[Any]       =    os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""vocab_file"""]						)
         with open(self.vocab_file       ,				"""w"""       ,				encoding="""utf-8"""						) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)
  
  
       def 					__snake_case(     self       ,				A_						):
         _UpperCAmelCase		:       Tuple       =    """UNwant\u00E9d,running"""
         _UpperCAmelCase		:       str       =    """unwanted, running"""
         return input_text, output_text
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       str       =    self.tokenizer_class(self.vocab_file						)
  
         _UpperCAmelCase		:       Tuple       =    tokenizer.tokenize("""UNwant\u00E9d,running"""						)
         self.assertListEqual(A_       ,				["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]						)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(A_						)       ,				[9, 6, 7, 12, 10, 11]						)
  
  
       def 					__snake_case(     self						):
  
  
  
  
  
  
  
         if not self.test_rust_tokenizer:
           return
  
         _UpperCAmelCase		:       str       =    self.get_tokenizer()
         _UpperCAmelCase		:       Union[str, Any]       =    self.get_rust_tokenizer()
  
         _UpperCAmelCase		:       Tuple       =    """UNwant\u00E9d,running"""
  
         _UpperCAmelCase		:       List[str]       =    tokenizer.tokenize(A_						)
         _UpperCAmelCase		:       List[Any]       =    rust_tokenizer.tokenize(A_						)
         self.assertListEqual(A_       ,				A_						)
  
         _UpperCAmelCase		:       Optional[Any]       =    tokenizer.encode(A_       ,				add_special_tokens=A_						)
         _UpperCAmelCase		:       int       =    rust_tokenizer.encode(A_       ,				add_special_tokens=A_						)
         self.assertListEqual(A_       ,				A_						)
  
         _UpperCAmelCase		:       Union[str, Any]       =    self.get_rust_tokenizer()
         _UpperCAmelCase		:       List[str]       =    tokenizer.encode(A_						)
         _UpperCAmelCase		:       Union[str, Any]       =    rust_tokenizer.encode(A_						)
         self.assertListEqual(A_       ,				A_						)
  
         # With lower casing
         _UpperCAmelCase		:       Optional[Any]       =    self.get_tokenizer(do_lower_case=A_						)
         _UpperCAmelCase		:       List[str]       =    self.get_rust_tokenizer(do_lower_case=A_						)
  
         _UpperCAmelCase		:       Dict       =    """UNwant\u00E9d,running"""
  
         _UpperCAmelCase		:       List[Any]       =    tokenizer.tokenize(A_						)
         _UpperCAmelCase		:       Any       =    rust_tokenizer.tokenize(A_						)
         self.assertListEqual(A_       ,				A_						)
  
         _UpperCAmelCase		:       Tuple       =    tokenizer.encode(A_       ,				add_special_tokens=A_						)
         _UpperCAmelCase		:       List[str]       =    rust_tokenizer.encode(A_       ,				add_special_tokens=A_						)
         self.assertListEqual(A_       ,				A_						)
  
         _UpperCAmelCase		:       int       =    self.get_rust_tokenizer()
         _UpperCAmelCase		:       Union[str, Any]       =    tokenizer.encode(A_						)
         _UpperCAmelCase		:       List[Any]       =    rust_tokenizer.encode(A_						)
         self.assertListEqual(A_       ,				A_						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Tuple       =    BasicTokenizer()
  
         self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""						)       ,				["""ah""", """\u535A""", """\u63A8""", """zz"""]						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       str       =    BasicTokenizer(do_lower_case=A_						)
  
         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """						)       ,				["""hello""", """!""", """how""", """are""", """you""", """?"""]						)
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)       ,				["""hello"""]						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       List[Any]       =    BasicTokenizer(do_lower_case=A_       ,				strip_accents=A_						)
  
         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)       ,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]						)
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)       ,				["""h\u00E9llo"""]						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       str       =    BasicTokenizer(do_lower_case=A_       ,				strip_accents=A_						)
  
         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)       ,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]						)
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)       ,				["""hello"""]						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Union[str, Any]       =    BasicTokenizer(do_lower_case=A_						)
  
         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)       ,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]						)
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)       ,				["""hello"""]						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Tuple       =    BasicTokenizer(do_lower_case=A_						)
  
         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """						)       ,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Dict       =    BasicTokenizer(do_lower_case=A_       ,				strip_accents=A_						)
  
         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)       ,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Dict       =    BasicTokenizer(do_lower_case=A_       ,				strip_accents=A_						)
  
         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)       ,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       List[Any]       =    BasicTokenizer(do_lower_case=A_       ,				never_split=["""[UNK]"""]						)
  
         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""						)       ,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       List[Any]       =    ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]
  
         _UpperCAmelCase		:       int       =    {}
         for i, token in enumerate(A_						):
           _UpperCAmelCase		:       Union[str, Any]       =    i
         _UpperCAmelCase		:       Tuple       =    WordpieceTokenizer(vocab=A_       ,				unk_token="""[UNK]"""						)
  
         self.assertListEqual(tokenizer.tokenize(""""""						)       ,				[]						)
  
         self.assertListEqual(tokenizer.tokenize("""unwanted running"""						)       ,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]						)
  
         self.assertListEqual(tokenizer.tokenize("""unwantedX running"""						)       ,				["""[UNK]""", """runn""", """##ing"""]						)
  
  
       def 					__snake_case(     self						):
         self.assertTrue(_is_whitespace(""" """						)						)
         self.assertTrue(_is_whitespace("""\t"""						)						)
         self.assertTrue(_is_whitespace("""\r"""						)						)
         self.assertTrue(_is_whitespace("""\n"""						)						)
         self.assertTrue(_is_whitespace("""\u00A0"""						)						)
  
         self.assertFalse(_is_whitespace("""A"""						)						)
         self.assertFalse(_is_whitespace("""-"""						)						)
  
  
       def 					__snake_case(     self						):
         self.assertTrue(_is_control("""\u0005"""						)						)
  
         self.assertFalse(_is_control("""A"""						)						)
         self.assertFalse(_is_control(""" """						)						)
         self.assertFalse(_is_control("""\t"""						)						)
         self.assertFalse(_is_control("""\r"""						)						)
  
  
       def 					__snake_case(     self						):
         self.assertTrue(_is_punctuation("""-"""						)						)
         self.assertTrue(_is_punctuation("""$"""						)						)
         self.assertTrue(_is_punctuation("""`"""						)						)
         self.assertTrue(_is_punctuation("""."""						)						)
  
         self.assertFalse(_is_punctuation("""A"""						)						)
         self.assertFalse(_is_punctuation(""" """						)						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Optional[int]       =    self.get_tokenizer()
  
         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(A_						) for t in ["""Test""", """\xad""", """test"""]]       ,				[["""[UNK]"""], [], ["""[UNK]"""]]						)
  
         if self.test_rust_tokenizer:
           _UpperCAmelCase		:       Tuple       =    self.get_rust_tokenizer()
           self.assertListEqual(
               [rust_tokenizer.tokenize(A_						) for t in ["""Test""", """\xad""", """test"""]]       ,				[["""[UNK]"""], [], ["""[UNK]"""]]						)
  
  
       @slow
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       int       =    self.tokenizer_class.from_pretrained("""bert-base-uncased"""						)
  
         _UpperCAmelCase		:       str       =    tokenizer.encode("""sequence builders"""       ,				add_special_tokens=A_						)
         _UpperCAmelCase		:       int       =    tokenizer.encode("""multi-sequence build"""       ,				add_special_tokens=A_						)
  
         _UpperCAmelCase		:       List[Any]       =    tokenizer.build_inputs_with_special_tokens(A_						)
         _UpperCAmelCase		:       int       =    tokenizer.build_inputs_with_special_tokens(A_       ,				A_						)
  
         assert encoded_sentence == [1_01] + text + [1_02]
         assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
  
  
       def 					__snake_case(     self						):
  
  
  
  
  
  
  
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
             _UpperCAmelCase		:       Union[str, Any]       =    self.rust_tokenizer_class.from_pretrained(A_       ,				**A_						)
  
             _UpperCAmelCase		:       Union[str, Any]       =    F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
             _UpperCAmelCase		:       Any       =    tokenizer_r.encode_plus(
                 A_       ,				return_attention_mask=A_       ,				return_token_type_ids=A_       ,				return_offsets_mapping=A_       ,				add_special_tokens=A_       ,				)
  
             _UpperCAmelCase		:       List[Any]       =    tokenizer_r.do_lower_case if hasattr(A_       ,				"""do_lower_case"""						) else False
             _UpperCAmelCase		:       Dict       =    (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), """A"""),
                     ((1, 2), ""","""),
                     ((3, 5), """na"""),
                     ((5, 6), """##ï"""),
                     ((6, 8), """##ve"""),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), """Allen"""),
                     ((21, 23), """##NL"""),
                     ((23, 24), """##P"""),
                     ((25, 33), """sentence"""),
                     ((33, 34), """."""),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), """a"""),
                     ((1, 2), ""","""),
                     ((3, 8), """naive"""),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), """allen"""),
                     ((21, 23), """##nl"""),
                     ((23, 24), """##p"""),
                     ((25, 33), """sentence"""),
                     ((33, 34), """."""),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )
  
             self.assertEqual(
                 [e[1] for e in expected_results]       ,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]						)						)
             self.assertEqual([e[0] for e in expected_results]       ,				tokens["""offset_mapping"""]						)
  
  
  
  
       @slow
       def 					__snake_case(     self						):
  
  
  
  
  
  
  
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
             _UpperCAmelCase		:       Dict       =    self.rust_tokenizer_class.from_pretrained(A_       ,				**A_						)
             _UpperCAmelCase		:       int       =    self.tokenizer_class.from_pretrained(A_       ,				**A_						)
             _UpperCAmelCase		:       str       =    [["""Hello world!""", """Nice to meet you!"""], ["""The cute cat.""", """The adorable dog."""]]
  
             _UpperCAmelCase		:       List[Any]       =    tokenizer_r.batch_encode_candidates(A_       ,				max_length=10       ,				return_tensors="""np"""						)
             _UpperCAmelCase		:       Union[str, Any]       =    tokenizer_p.batch_encode_candidates(A_       ,				max_length=10       ,				return_tensors="""np"""						)
  
             _UpperCAmelCase		:       Optional[int]       =    (2, 2, 10)
  
             self.assertEqual(encoded_sentence_r["""input_ids"""].shape       ,				A_						)
             self.assertEqual(encoded_sentence_r["""attention_mask"""].shape       ,				A_						)
             self.assertEqual(encoded_sentence_r["""token_type_ids"""].shape       ,				A_						)
  
             self.assertEqual(encoded_sentence_p["""input_ids"""].shape       ,				A_						)
             self.assertEqual(encoded_sentence_p["""attention_mask"""].shape       ,				A_						)
             self.assertEqual(encoded_sentence_p["""token_type_ids"""].shape       ,				A_						)
  
  
  
  
  
  
style_context: 
  
  
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils import AddedToken
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  
  
  if is_sentencepiece_available():
   from .tokenization_xlm_roberta import XLMRobertaTokenizer
  else:
   SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    None
  
  
  SCREAMING_SNAKE_CASE__				:    Dict         =    logging.get_logger(__name__)
  
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}
  
  SCREAMING_SNAKE_CASE__				:    Any         =    {
      'vocab_file': {
          'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
          'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
          'xlm-roberta-large-finetuned-conll02-dutch': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
          ),
          'xlm-roberta-large-finetuned-conll02-spanish': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
          ),
          'xlm-roberta-large-finetuned-conll03-english': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
          ),
          'xlm-roberta-large-finetuned-conll03-german': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
          ),
      },
      'tokenizer_file': {
          'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
          'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
          'xlm-roberta-large-finetuned-conll02-dutch': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
          ),
          'xlm-roberta-large-finetuned-conll02-spanish': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
          ),
          'xlm-roberta-large-finetuned-conll03-english': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
          ),
          'xlm-roberta-large-finetuned-conll03-german': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
          ),
      },
  }
  
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    {
      'xlm-roberta-base': 512,
      'xlm-roberta-large': 512,
      'xlm-roberta-large-finetuned-conll02-dutch': 512,
      'xlm-roberta-large-finetuned-conll02-spanish': 512,
      'xlm-roberta-large-finetuned-conll03-english': 512,
      'xlm-roberta-large-finetuned-conll03-german': 512,
  }
  
  
  class 		_SCREAMING_SNAKE_CASE    (	A    ):
       __SCREAMING_SNAKE_CASE    				=						VOCAB_FILES_NAMES
       __SCREAMING_SNAKE_CASE    				=						PRETRAINED_VOCAB_FILES_MAP
       __SCREAMING_SNAKE_CASE    				=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __SCREAMING_SNAKE_CASE    				=						['''input_ids''', '''attention_mask''']
       __SCREAMING_SNAKE_CASE    				=						XLMRobertaTokenizer
  
  
       def __init__(     self       ,				A_=None       ,				A_=None       ,				A_="<s>"       ,				A_="</s>"       ,				A_="</s>"       ,				A_="<s>"       ,				A_="<unk>"       ,				A_="<pad>"       ,				A_="<mask>"       ,				**A_       ,				):
         # Mask token behave like a normal word, i.e. include the space before it
         _UpperCAmelCase		:       int       =    AddedToken(A_       ,				lstrip=A_       ,				rstrip=A_						) if isinstance(A_       ,				A_						) else mask_token
  
         super().__init__(
             A_       ,				tokenizer_file=A_       ,				bos_token=A_       ,				eos_token=A_       ,				sep_token=A_       ,				cls_token=A_       ,				unk_token=A_       ,				pad_token=A_       ,				mask_token=A_       ,				**A_       ,				)
  
         _UpperCAmelCase		:       Union[str, Any]       =    vocab_file
         _UpperCAmelCase		:       List[str]       =    False if not self.vocab_file else True
  
  
       def 					__snake_case(     self       ,				A_       ,				A_ = None						):
  
         if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         _UpperCAmelCase		:       Optional[Any]       =    [self.cls_token_id]
         _UpperCAmelCase		:       Union[str, Any]       =    [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep
  
  
       def 					__snake_case(     self       ,				A_       ,				A_ = None						):
         _UpperCAmelCase		:       Tuple       =    [self.sep_token_id]
         _UpperCAmelCase		:       Optional[int]       =    [self.cls_token_id]
  
         if token_ids_a is None:
           return len(cls + token_ids_a + sep						) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]
  
  
  
  
       def 					__snake_case(     self       ,				A_       ,				A_ = None						):
         if not self.can_save_slow_tokenizer:
           raise ValueError(
               """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
               """tokenizer."""						)
  
         if not os.path.isdir(A_						):
           logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''						)
           return
         _UpperCAmelCase		:       List[str]       =    os.path.join(
             A_       ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
  
         if os.path.abspath(self.vocab_file						) != os.path.abspath(A_						):
           copyfile(self.vocab_file       ,				A_						)
  
         return (out_vocab_file,)
  
  
  
  
  
  