




import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 							__lowercase   (			a__	,		unittest.TestCase    ):
     _lowerCAmelCase		       =						RealmTokenizer
     _lowerCAmelCase		       =						RealmTokenizerFast
     _lowerCAmelCase		       =						True
     _lowerCAmelCase		       =						True
     _lowerCAmelCase		       =						filter_non_english



     def 				__magic_name__		(  self   :							Optional[int]       ):
       super().setUp()

       a_		    =  [
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       a_		    =  os.path.join(self.tmpdirname      ,		VOCAB_FILES_NAMES['''vocab_file''']       )
       with open(self.vocab_file      ,		'''w'''      ,		encoding='''utf-8'''       ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )



     def 				__magic_name__		(  self   :							Union[str, Any]      ,		lowercase__   :							str       ):
       a_		    =  '''UNwant\u00E9d,running'''
       a_		    =  '''unwanted, running'''
       return input_text, output_text



     def 				__magic_name__		(  self   :							Dict       ):
       a_		    =  self.tokenizer_class(self.vocab_file       )

       a_		    =  tokenizer.tokenize('''UNwant\u00E9d,running'''       )
       self.assertListEqual(lowercase__      ,		['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']       )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )      ,		[9, 6, 7, 1_2, 1_0, 1_1]       )



     def 				__magic_name__		(  self   :							Optional[Any]       ):
       if not self.test_rust_tokenizer:
         return

       a_		    =  self.get_tokenizer()
       a_		    =  self.get_rust_tokenizer()

       a_		    =  '''UNwant\u00E9d,running'''

       a_		    =  tokenizer.tokenize(lowercase__       )
       a_		    =  rust_tokenizer.tokenize(lowercase__       )
       self.assertListEqual(lowercase__      ,		lowercase__       )

       a_		    =  tokenizer.encode(lowercase__      ,		add_special_tokens=lowercase__       )
       a_		    =  rust_tokenizer.encode(lowercase__      ,		add_special_tokens=lowercase__       )
       self.assertListEqual(lowercase__      ,		lowercase__       )

       a_		    =  self.get_rust_tokenizer()
       a_		    =  tokenizer.encode(lowercase__       )
       a_		    =  rust_tokenizer.encode(lowercase__       )
       self.assertListEqual(lowercase__      ,		lowercase__       )

       # With lower casing
       a_		    =  self.get_tokenizer(do_lower_case=lowercase__       )
       a_		    =  self.get_rust_tokenizer(do_lower_case=lowercase__       )

       a_		    =  '''UNwant\u00E9d,running'''

       a_		    =  tokenizer.tokenize(lowercase__       )
       a_		    =  rust_tokenizer.tokenize(lowercase__       )
       self.assertListEqual(lowercase__      ,		lowercase__       )

       a_		    =  tokenizer.encode(lowercase__      ,		add_special_tokens=lowercase__       )
       a_		    =  rust_tokenizer.encode(lowercase__      ,		add_special_tokens=lowercase__       )
       self.assertListEqual(lowercase__      ,		lowercase__       )

       a_		    =  self.get_rust_tokenizer()
       a_		    =  tokenizer.encode(lowercase__       )
       a_		    =  rust_tokenizer.encode(lowercase__       )
       self.assertListEqual(lowercase__      ,		lowercase__       )



     def 				__magic_name__		(  self   :							Optional[int]       ):
       a_		    =  BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''       )      ,		['''ah''', '''\u535A''', '''\u63A8''', '''zz''']       )



     def 				__magic_name__		(  self   :							int       ):
       a_		    =  BasicTokenizer(do_lower_case=lowercase__       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )      ,		['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )      ,		['''hello''']       )



     def 				__magic_name__		(  self   :							Dict       ):
       a_		    =  BasicTokenizer(do_lower_case=lowercase__      ,		strip_accents=lowercase__       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )      ,		['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )      ,		['''h\u00E9llo''']       )



     def 				__magic_name__		(  self   :							List[str]       ):
       a_		    =  BasicTokenizer(do_lower_case=lowercase__      ,		strip_accents=lowercase__       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )      ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )      ,		['''hello''']       )



     def 				__magic_name__		(  self   :							Dict       ):
       a_		    =  BasicTokenizer(do_lower_case=lowercase__       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )      ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )      ,		['''hello''']       )



     def 				__magic_name__		(  self   :							Optional[Any]       ):
       a_		    =  BasicTokenizer(do_lower_case=lowercase__       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )      ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )



     def 				__magic_name__		(  self   :							Tuple       ):
       a_		    =  BasicTokenizer(do_lower_case=lowercase__      ,		strip_accents=lowercase__       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )      ,		['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )



     def 				__magic_name__		(  self   :							Any       ):
       a_		    =  BasicTokenizer(do_lower_case=lowercase__      ,		strip_accents=lowercase__       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )      ,		['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )



     def 				__magic_name__		(  self   :							Union[str, Any]       ):
       a_		    =  BasicTokenizer(do_lower_case=lowercase__      ,		never_split=['''[UNK]''']       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''       )      ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']       )



     def 				__magic_name__		(  self   :							Dict       ):
       a_		    =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       a_		    =  {}
       for i, token in enumerate(lowercase__       ):
         a_		    =  i
       a_		    =  WordpieceTokenizer(vocab=lowercase__      ,		unk_token='''[UNK]'''       )

       self.assertListEqual(tokenizer.tokenize(''''''       )      ,		[]       )

       self.assertListEqual(tokenizer.tokenize('''unwanted running'''       )      ,		['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']       )

       self.assertListEqual(tokenizer.tokenize('''unwantedX running'''       )      ,		['''[UNK]''', '''runn''', '''##ing''']       )



     def 				__magic_name__		(  self   :							Any       ):
       self.assertTrue(_is_whitespace(''' '''       )       )
       self.assertTrue(_is_whitespace('''\t'''       )       )
       self.assertTrue(_is_whitespace('''\r'''       )       )
       self.assertTrue(_is_whitespace('''\n'''       )       )
       self.assertTrue(_is_whitespace('''\u00A0'''       )       )

       self.assertFalse(_is_whitespace('''A'''       )       )
       self.assertFalse(_is_whitespace('''-'''       )       )



     def 				__magic_name__		(  self   :							Dict       ):
       self.assertTrue(_is_control('''\u0005'''       )       )

       self.assertFalse(_is_control('''A'''       )       )
       self.assertFalse(_is_control(''' '''       )       )
       self.assertFalse(_is_control('''\t'''       )       )
       self.assertFalse(_is_control('''\r'''       )       )



     def 				__magic_name__		(  self   :							Dict       ):
       self.assertTrue(_is_punctuation('''-'''       )       )
       self.assertTrue(_is_punctuation('''$'''       )       )
       self.assertTrue(_is_punctuation('''`'''       )       )
       self.assertTrue(_is_punctuation('''.'''       )       )

       self.assertFalse(_is_punctuation('''A'''       )       )
       self.assertFalse(_is_punctuation(''' '''       )       )



     def 				__magic_name__		(  self   :							Union[str, Any]       ):
       a_		    =  self.get_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(lowercase__       ) for t in ['''Test''', '''\xad''', '''test''']]      ,		[['''[UNK]'''], [], ['''[UNK]''']]       )

       if self.test_rust_tokenizer:
         a_		    =  self.get_rust_tokenizer()
         self.assertListEqual(
             [rust_tokenizer.tokenize(lowercase__       ) for t in ['''Test''', '''\xad''', '''test''']]      ,		[['''[UNK]'''], [], ['''[UNK]''']]       )



     @slow
     def 				__magic_name__		(  self   :							Union[str, Any]       ):
       a_		    =  self.tokenizer_class.from_pretrained('''bert-base-uncased'''       )

       a_		    =  tokenizer.encode('''sequence builders'''      ,		add_special_tokens=lowercase__       )
       a_		    =  tokenizer.encode('''multi-sequence build'''      ,		add_special_tokens=lowercase__       )

       a_		    =  tokenizer.build_inputs_with_special_tokens(lowercase__       )
       a_		    =  tokenizer.build_inputs_with_special_tokens(lowercase__      ,		lowercase__       )

       assert encoded_sentence == [1_0_1] + text + [1_0_2]
       assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]



     def 				__magic_name__		(  self   :							List[str]       ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
           a_		    =  self.rust_tokenizer_class.from_pretrained(lowercase__      ,		**lowercase__       )

           a_		    =  f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
           a_		    =  tokenizer_r.encode_plus(
               lowercase__      ,		return_attention_mask=lowercase__      ,		return_token_type_ids=lowercase__      ,		return_offsets_mapping=lowercase__      ,		add_special_tokens=lowercase__      ,		)

           a_		    =  tokenizer_r.do_lower_case if hasattr(lowercase__      ,		'''do_lower_case'''       ) else False
           a_		    =  (
               [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''A'''),
                   ((1, 2), ''','''),
                   ((3, 5), '''na'''),
                   ((5, 6), '''##ï'''),
                   ((6, 8), '''##ve'''),
                   ((9, 1_5), tokenizer_r.mask_token),
                   ((1_6, 2_1), '''Allen'''),
                   ((2_1, 2_3), '''##NL'''),
                   ((2_3, 2_4), '''##P'''),
                   ((2_5, 3_3), '''sentence'''),
                   ((3_3, 3_4), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
               if not do_lower_case
               else [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''a'''),
                   ((1, 2), ''','''),
                   ((3, 8), '''naive'''),
                   ((9, 1_5), tokenizer_r.mask_token),
                   ((1_6, 2_1), '''allen'''),
                   ((2_1, 2_3), '''##nl'''),
                   ((2_3, 2_4), '''##p'''),
                   ((2_5, 3_3), '''sentence'''),
                   ((3_3, 3_4), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
           )

           self.assertEqual(
               [e[1] for e in expected_results]      ,		tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']       )       )
           self.assertEqual([e[0] for e in expected_results]      ,		tokens['''offset_mapping''']       )



     @slow
     def 				__magic_name__		(  self   :							int       ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
           a_		    =  self.rust_tokenizer_class.from_pretrained(lowercase__      ,		**lowercase__       )
           a_		    =  self.tokenizer_class.from_pretrained(lowercase__      ,		**lowercase__       )
           a_		    =  [['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

           a_		    =  tokenizer_r.batch_encode_candidates(lowercase__      ,		max_length=1_0      ,		return_tensors='''np'''       )
           a_		    =  tokenizer_p.batch_encode_candidates(lowercase__      ,		max_length=1_0      ,		return_tensors='''np'''       )

           a_		    =  (2, 2, 1_0)

           self.assertEqual(encoded_sentence_r['''input_ids'''].shape      ,		lowercase__       )
           self.assertEqual(encoded_sentence_r['''attention_mask'''].shape      ,		lowercase__       )
           self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape      ,		lowercase__       )

           self.assertEqual(encoded_sentence_p['''input_ids'''].shape      ,		lowercase__       )
           self.assertEqual(encoded_sentence_p['''attention_mask'''].shape      ,		lowercase__       )
           self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape      ,		lowercase__       )






from __future__ import annotations

from typing import Any






def 				UpperCAmelCase__  (							*_A			):






  """simple docstring"""


  if all(_check_not_integer(_A			) for m in matrix_s			):
    for i in matrix_s[1:]:
      _verify_matrix_sizes(matrix_s[0]      ,      _A			)
    return [[sum(_A			) for t in zip(*_A			)] for m in zip(*_A			)]
  raise TypeError('''Expected a matrix, got int/list instead'''			)






def 				UpperCAmelCase__  (							_A      ,      _A			):






  """simple docstring"""


  if (
      _check_not_integer(_A			)
      and _check_not_integer(_A			)
      and _verify_matrix_sizes(_A      ,      _A			)
  ):
    return [[i - j for i, j in zip(*_A			)] for m in zip(_A      ,      _A			)]
  raise TypeError('''Expected a matrix, got int/list instead'''			)






def 				UpperCAmelCase__  (							_A      ,      _A			):






  """simple docstring"""


  return [[x * n for x in row] for row in matrix]






def 				UpperCAmelCase__  (							_A      ,      _A			):






  """simple docstring"""


  if _check_not_integer(_A			) and _check_not_integer(_A			):
    a_    , a_		    =  _verify_matrix_sizes(_A      ,      _A			)

  if cols[0] != rows[1]:
    a_		    =  (
        '''Cannot multiply matrix of dimensions '''
        f"({rows[0]},{cols[0]}) and ({rows[1]},{cols[1]})"
    )
    raise ValueError(_A			)
  return [
      [sum(m * n for m, n in zip(_A      ,      _A			)			) for j in zip(*_A			)] for i in matrix_a
  ]






def 				UpperCAmelCase__  (							_A			):






  """simple docstring"""


  a_		    =  int(_A			)
  return [[int(row == column			) for column in range(_A			)] for row in range(_A			)]






def 				UpperCAmelCase__  (							_A      ,      _A = True			):






  """simple docstring"""


  if _check_not_integer(_A			):
    if return_map:
      return map(_A      ,      zip(*_A			)			)
    else:
      return list(map(_A      ,      zip(*_A			)			)			)
  raise TypeError('''Expected a matrix, got int/list instead'''			)






def 				UpperCAmelCase__  (							_A      ,      _A      ,      _A			):






  """simple docstring"""


  a_		    =  matrix[:row] + matrix[row + 1 :]
  return [row[:column] + row[column + 1 :] for row in minor]






def 				UpperCAmelCase__  (							_A			):






  """simple docstring"""


  if len(_A			) == 1:
    return matrix[0][0]

  return sum(
      x * determinant(minor(_A      ,      0      ,      _A			)			) * (-1) ** i
      for i, x in enumerate(matrix[0]			)			)






def 				UpperCAmelCase__  (							_A			):






  """simple docstring"""


  a_		    =  determinant(_A			)
  if det == 0:
    return None

  a_		    =  [
      [determinant(minor(_A      ,      _A      ,      _A			)			) for j in range(len(_A			)			)]
      for i in range(len(_A			)			)
  ]

  a_		    =  [
      [x * (-1) ** (row + col) for col, x in enumerate(matrix_minor[row]			)]
      for row in range(len(_A			)			)
  ]
  a_		    =  list(transpose(_A			)			)
  return scalar_multiply(_A      ,      1 / det			)






def 				UpperCAmelCase__  (							_A			):






  """simple docstring"""


  return not isinstance(_A      ,      _A			) and not isinstance(matrix[0]      ,      _A			)






def 				UpperCAmelCase__  (							_A			):






  """simple docstring"""


  return len(_A			), len(matrix[0]			)






def 				UpperCAmelCase__  (							_A      ,      _A			):






  """simple docstring"""


  a_		    =  _shape(_A			) + _shape(_A			)
  if shape[0] != shape[3] or shape[1] != shape[2]:
    a_		    =  (
        '''operands could not be broadcast together with shape '''
        f"({shape[0], shape[1]}), ({shape[2], shape[3]})"
    )
    raise ValueError(_A			)
  return (shape[0], shape[2]), (shape[1], shape[3])






def 				UpperCAmelCase__  (							):






  """simple docstring"""


  a_		    =  [[12, 10], [3, 9]]
  a_		    =  [[3, 4], [7, 4]]
  a_		    =  [[11, 12, 13, 14], [21, 22, 23, 24], [31, 32, 33, 34], [41, 42, 43, 44]]
  a_		    =  [[3, 0, 2], [2, 0, -2], [0, 1, 1]]
  print(f"Add Operation, {add(_A      ,      _A			) = } \n"			)
  print(
      f"Multiply Operation, {multiply(_A      ,      _A			) = } \n"      ,      )
  print(f"Identity: {identity(5			)}\n"			)
  print(f"Minor of {matrix_c} = {minor(_A      ,      1      ,      2			)} \n"			)
  print(f"Determinant of {matrix_b} = {determinant(_A			)} \n"			)
  print(f"Inverse of {matrix_d} = {inverse(_A			)}\n"			)


if __name__ == "__main__":
  import doctest

  doctest.testmod()
  main()
