



"""simple docstring"""






import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class    __snake_case(					_A			,			unittest.TestCase       ):
    _A       = RealmTokenizer
    _A       = RealmTokenizerFast
    _A       = True
    _A       = True
    _A       = filter_non_english




    def   A   (   self			):





      '''simple docstring'''



      super().setUp()

      _SCREAMING_SNAKE_CASE        =			[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      _SCREAMING_SNAKE_CASE        =			os.path.join(self.tmpdirname    ,							VOCAB_FILES_NAMES['''vocab_file''']			)
      with open(self.vocab_file    ,							'''w'''    ,							encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)




    def   A   (   self    ,							A_			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			'UNwant\u00E9d,running'
      _SCREAMING_SNAKE_CASE        =			'unwanted, running'
      return input_text, output_text




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			self.tokenizer_class(self.vocab_file			)

      _SCREAMING_SNAKE_CASE        =			tokenizer.tokenize('''UNwant\u00E9d,running'''			)
      self.assertListEqual(_UpperCAmelCase    ,							['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']			)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase			)    ,							[9, 6, 7, 12, 10, 11]			)




    def   A   (   self			):





      '''simple docstring'''



      if not self.test_rust_tokenizer:
        return

      _SCREAMING_SNAKE_CASE        =			self.get_tokenizer()
      _SCREAMING_SNAKE_CASE        =			self.get_rust_tokenizer()

      _SCREAMING_SNAKE_CASE        =			'UNwant\u00E9d,running'

      _SCREAMING_SNAKE_CASE        =			tokenizer.tokenize(_UpperCAmelCase			)
      _SCREAMING_SNAKE_CASE        =			rust_tokenizer.tokenize(_UpperCAmelCase			)
      self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase			)

      _SCREAMING_SNAKE_CASE        =			tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase			)
      _SCREAMING_SNAKE_CASE        =			rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase			)
      self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase			)

      _SCREAMING_SNAKE_CASE        =			self.get_rust_tokenizer()
      _SCREAMING_SNAKE_CASE        =			tokenizer.encode(_UpperCAmelCase			)
      _SCREAMING_SNAKE_CASE        =			rust_tokenizer.encode(_UpperCAmelCase			)
      self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase			)

      # With lower casing
      _SCREAMING_SNAKE_CASE        =			self.get_tokenizer(do_lower_case=_UpperCAmelCase			)
      _SCREAMING_SNAKE_CASE        =			self.get_rust_tokenizer(do_lower_case=_UpperCAmelCase			)

      _SCREAMING_SNAKE_CASE        =			'UNwant\u00E9d,running'

      _SCREAMING_SNAKE_CASE        =			tokenizer.tokenize(_UpperCAmelCase			)
      _SCREAMING_SNAKE_CASE        =			rust_tokenizer.tokenize(_UpperCAmelCase			)
      self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase			)

      _SCREAMING_SNAKE_CASE        =			tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase			)
      _SCREAMING_SNAKE_CASE        =			rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase			)
      self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase			)

      _SCREAMING_SNAKE_CASE        =			self.get_rust_tokenizer()
      _SCREAMING_SNAKE_CASE        =			tokenizer.encode(_UpperCAmelCase			)
      _SCREAMING_SNAKE_CASE        =			rust_tokenizer.encode(_UpperCAmelCase			)
      self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''			)    ,							['''ah''', '''\u535A''', '''\u63A8''', '''zz''']			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			BasicTokenizer(do_lower_case=_UpperCAmelCase			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)    ,							['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)    ,							['''hello''']			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			BasicTokenizer(do_lower_case=_UpperCAmelCase    ,							strip_accents=_UpperCAmelCase			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,							['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)    ,							['''h\u00E9llo''']			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			BasicTokenizer(do_lower_case=_UpperCAmelCase    ,							strip_accents=_UpperCAmelCase			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)    ,							['''hello''']			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			BasicTokenizer(do_lower_case=_UpperCAmelCase			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)    ,							['''hello''']			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			BasicTokenizer(do_lower_case=_UpperCAmelCase			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)    ,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			BasicTokenizer(do_lower_case=_UpperCAmelCase    ,							strip_accents=_UpperCAmelCase			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,							['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			BasicTokenizer(do_lower_case=_UpperCAmelCase    ,							strip_accents=_UpperCAmelCase			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,							['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			BasicTokenizer(do_lower_case=_UpperCAmelCase    ,							never_split=['''[UNK]''']			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''			)    ,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      _SCREAMING_SNAKE_CASE        =			{}
      for i, token in enumerate(_UpperCAmelCase			):
        _SCREAMING_SNAKE_CASE        =			i
      _SCREAMING_SNAKE_CASE        =			WordpieceTokenizer(vocab=_UpperCAmelCase    ,							unk_token='''[UNK]'''			)

      self.assertListEqual(tokenizer.tokenize(''''''			)    ,							[]			)

      self.assertListEqual(tokenizer.tokenize('''unwanted running'''			)    ,							['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']			)

      self.assertListEqual(tokenizer.tokenize('''unwantedX running'''			)    ,							['''[UNK]''', '''runn''', '''##ing''']			)




    def   A   (   self			):





      '''simple docstring'''



      self.assertTrue(_is_whitespace(''' '''			)			)
      self.assertTrue(_is_whitespace('''\t'''			)			)
      self.assertTrue(_is_whitespace('''\r'''			)			)
      self.assertTrue(_is_whitespace('''\n'''			)			)
      self.assertTrue(_is_whitespace('''\u00A0'''			)			)

      self.assertFalse(_is_whitespace('''A'''			)			)
      self.assertFalse(_is_whitespace('''-'''			)			)




    def   A   (   self			):





      '''simple docstring'''



      self.assertTrue(_is_control('''\u0005'''			)			)

      self.assertFalse(_is_control('''A'''			)			)
      self.assertFalse(_is_control(''' '''			)			)
      self.assertFalse(_is_control('''\t'''			)			)
      self.assertFalse(_is_control('''\r'''			)			)




    def   A   (   self			):





      '''simple docstring'''



      self.assertTrue(_is_punctuation('''-'''			)			)
      self.assertTrue(_is_punctuation('''$'''			)			)
      self.assertTrue(_is_punctuation('''`'''			)			)
      self.assertTrue(_is_punctuation('''.'''			)			)

      self.assertFalse(_is_punctuation('''A'''			)			)
      self.assertFalse(_is_punctuation(''' '''			)			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			self.get_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(_UpperCAmelCase			) for t in ['''Test''', '''\xad''', '''test''']]    ,							[['''[UNK]'''], [], ['''[UNK]''']]			)

      if self.test_rust_tokenizer:
        _SCREAMING_SNAKE_CASE        =			self.get_rust_tokenizer()
        self.assertListEqual(
            [rust_tokenizer.tokenize(_UpperCAmelCase			) for t in ['''Test''', '''\xad''', '''test''']]    ,							[['''[UNK]'''], [], ['''[UNK]''']]			)




    @slow
    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			self.tokenizer_class.from_pretrained('''bert-base-uncased'''			)

      _SCREAMING_SNAKE_CASE        =			tokenizer.encode('''sequence builders'''    ,							add_special_tokens=_UpperCAmelCase			)
      _SCREAMING_SNAKE_CASE        =			tokenizer.encode('''multi-sequence build'''    ,							add_special_tokens=_UpperCAmelCase			)

      _SCREAMING_SNAKE_CASE        =			tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase			)
      _SCREAMING_SNAKE_CASE        =			tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase    ,							_UpperCAmelCase			)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]




    def   A   (   self			):





      '''simple docstring'''



      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
          _SCREAMING_SNAKE_CASE        =			self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase    ,							**_UpperCAmelCase			)

          _SCREAMING_SNAKE_CASE        =			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
          _SCREAMING_SNAKE_CASE        =			tokenizer_r.encode_plus(
              _UpperCAmelCase    ,							return_attention_mask=_UpperCAmelCase    ,							return_token_type_ids=_UpperCAmelCase    ,							return_offsets_mapping=_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase    ,							)

          _SCREAMING_SNAKE_CASE        =			tokenizer_r.do_lower_case if hasattr(_UpperCAmelCase    ,							'''do_lower_case'''			) else False
          _SCREAMING_SNAKE_CASE        =			(
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'A'),
                  ((1, 2), ','),
                  ((3, 5), 'na'),
                  ((5, 6), '##ï'),
                  ((6, 8), '##ve'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'Allen'),
                  ((21, 23), '##NL'),
                  ((23, 24), '##P'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'a'),
                  ((1, 2), ','),
                  ((3, 8), 'naive'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'allen'),
                  ((21, 23), '##nl'),
                  ((23, 24), '##p'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results]    ,							tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']			)			)
          self.assertEqual([e[0] for e in expected_results]    ,							tokens['''offset_mapping''']			)




    @slow
    def   A   (   self			):





      '''simple docstring'''



      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
          _SCREAMING_SNAKE_CASE        =			self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase    ,							**_UpperCAmelCase			)
          _SCREAMING_SNAKE_CASE        =			self.tokenizer_class.from_pretrained(_UpperCAmelCase    ,							**_UpperCAmelCase			)
          _SCREAMING_SNAKE_CASE        =			[['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

          _SCREAMING_SNAKE_CASE        =			tokenizer_r.batch_encode_candidates(_UpperCAmelCase    ,							max_length=10    ,							return_tensors='''np'''			)
          _SCREAMING_SNAKE_CASE        =			tokenizer_p.batch_encode_candidates(_UpperCAmelCase    ,							max_length=10    ,							return_tensors='''np'''			)

          _SCREAMING_SNAKE_CASE        =			(2, 2, 10)

          self.assertEqual(encoded_sentence_r['''input_ids'''].shape    ,							_UpperCAmelCase			)
          self.assertEqual(encoded_sentence_r['''attention_mask'''].shape    ,							_UpperCAmelCase			)
          self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape    ,							_UpperCAmelCase			)

          self.assertEqual(encoded_sentence_p['''input_ids'''].shape    ,							_UpperCAmelCase			)
          self.assertEqual(encoded_sentence_p['''attention_mask'''].shape    ,							_UpperCAmelCase			)
          self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape    ,							_UpperCAmelCase			)




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


snake_case__  				=			{'configuration_qdqbert': ['QDQBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'QDQBertConfig']}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			snake_case__  				=			[
			    'QDQBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'QDQBertForMaskedLM',
			    'QDQBertForMultipleChoice',
			    'QDQBertForNextSentencePrediction',
			    'QDQBertForQuestionAnswering',
			    'QDQBertForSequenceClassification',
			    'QDQBertForTokenClassification',
			    'QDQBertLayer',
			    'QDQBertLMHeadModel',
			    'QDQBertModel',
			    'QDQBertPreTrainedModel',
			    'load_tf_weights_in_qdqbert',
			]


if TYPE_CHECKING:
			from .configuration_qdqbert import QDQBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, QDQBertConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_qdqbert import (
						    QDQBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    QDQBertForMaskedLM,
						    QDQBertForMultipleChoice,
						    QDQBertForNextSentencePrediction,
						    QDQBertForQuestionAnswering,
						    QDQBertForSequenceClassification,
						    QDQBertForTokenClassification,
						    QDQBertLayer,
						    QDQBertLMHeadModel,
						    QDQBertModel,
						    QDQBertPreTrainedModel,
						    load_tf_weights_in_qdqbert,
						)


else:
			import sys

			snake_case__  				=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


