

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 					A_   (      A__ ,     unittest.TestCase						):


    '''simple docstring'''






    a__     							=       ElectraTokenizer
    a__     							=       ElectraTokenizerFast
    a__     							=       True
    a__     							=       True
    a__     							=       filter_non_english


    def      lowerCAmelCase_						(self  )     ->      Union[str, Any]:
        super().setUp()

        __UpperCAmelCase									=			[
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''[PAD]''',
            '''[MASK]''',
            '''want''',
            '''##want''',
            '''##ed''',
            '''wa''',
            '''un''',
            '''runn''',
            '''##ing''',
            ''',''',
            '''low''',
            '''lowest''',
        ]
        __UpperCAmelCase									=			os.path.join(self.tmpdirname     ,      VOCAB_FILES_NAMES['''vocab_file''']  )
        with open(self.vocab_file     ,      '''w'''     ,      encoding='''utf-8'''  ) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )


    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      List[str]:
        __UpperCAmelCase									=			'''UNwant\u00E9d,running'''
        __UpperCAmelCase									=			'''unwanted, running'''
        return input_text, output_text


    def      lowerCAmelCase_						(self  )     ->      str:
        __UpperCAmelCase									=			self.tokenizer_class(self.vocab_file  )

        __UpperCAmelCase									=			tokenizer.tokenize('''UNwant\u00E9d,running'''  )
        self.assertListEqual(_lowerCAmelCase     ,      ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase  )     ,      [9, 6, 7, 12, 10, 11]  )


    def      lowerCAmelCase_						(self  )     ->      Dict:
        if not self.test_rust_tokenizer:
            return

        __UpperCAmelCase									=			self.get_tokenizer()
        __UpperCAmelCase									=			self.get_rust_tokenizer()

        __UpperCAmelCase									=			'''UNwant\u00E9d,running'''

        __UpperCAmelCase									=			tokenizer.tokenize(_lowerCAmelCase  )
        __UpperCAmelCase									=			rust_tokenizer.tokenize(_lowerCAmelCase  )
        self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )

        __UpperCAmelCase									=			tokenizer.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
        __UpperCAmelCase									=			rust_tokenizer.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
        self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )

        __UpperCAmelCase									=			self.get_rust_tokenizer()
        __UpperCAmelCase									=			tokenizer.encode(_lowerCAmelCase  )
        __UpperCAmelCase									=			rust_tokenizer.encode(_lowerCAmelCase  )
        self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )

        # With lower casing
        __UpperCAmelCase									=			self.get_tokenizer(do_lower_case=_lowerCAmelCase  )
        __UpperCAmelCase									=			self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase  )

        __UpperCAmelCase									=			'''UNwant\u00E9d,running'''

        __UpperCAmelCase									=			tokenizer.tokenize(_lowerCAmelCase  )
        __UpperCAmelCase									=			rust_tokenizer.tokenize(_lowerCAmelCase  )
        self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )

        __UpperCAmelCase									=			tokenizer.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
        __UpperCAmelCase									=			rust_tokenizer.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
        self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )

        __UpperCAmelCase									=			self.get_rust_tokenizer()
        __UpperCAmelCase									=			tokenizer.encode(_lowerCAmelCase  )
        __UpperCAmelCase									=			rust_tokenizer.encode(_lowerCAmelCase  )
        self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )


    def      lowerCAmelCase_						(self  )     ->      Optional[int]:
        __UpperCAmelCase									=			BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )     ,      ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )


    def      lowerCAmelCase_						(self  )     ->      Tuple:
        __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase  )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )     ,      ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )     ,      ['''hello''']  )


    def      lowerCAmelCase_						(self  )     ->      Any:
        __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      strip_accents=_lowerCAmelCase  )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )     ,      ['''h\u00E9llo''']  )


    def      lowerCAmelCase_						(self  )     ->      List[Any]:
        __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      strip_accents=_lowerCAmelCase  )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )     ,      ['''hello''']  )


    def      lowerCAmelCase_						(self  )     ->      Union[str, Any]:
        __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase  )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )     ,      ['''hello''']  )


    def      lowerCAmelCase_						(self  )     ->      Optional[int]:
        __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase  )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )     ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


    def      lowerCAmelCase_						(self  )     ->      Optional[int]:
        __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      strip_accents=_lowerCAmelCase  )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


    def      lowerCAmelCase_						(self  )     ->      str:
        __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      strip_accents=_lowerCAmelCase  )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


    def      lowerCAmelCase_						(self  )     ->      Optional[int]:
        __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      never_split=['''[UNK]''']  )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )     ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )


    def      lowerCAmelCase_						(self  )     ->      List[str]:
        __UpperCAmelCase									=			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

        __UpperCAmelCase									=			{}
        for i, token in enumerate(_lowerCAmelCase  ):
            __UpperCAmelCase									=			i
        __UpperCAmelCase									=			WordpieceTokenizer(vocab=_lowerCAmelCase     ,      unk_token='''[UNK]'''  )

        self.assertListEqual(tokenizer.tokenize(''''''  )     ,      []  )

        self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )     ,      ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

        self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )     ,      ['''[UNK]''', '''runn''', '''##ing''']  )


    def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
        self.assertTrue(_is_whitespace(''' '''  )  )
        self.assertTrue(_is_whitespace('''\t'''  )  )
        self.assertTrue(_is_whitespace('''\r'''  )  )
        self.assertTrue(_is_whitespace('''\n'''  )  )
        self.assertTrue(_is_whitespace('''\u00A0'''  )  )

        self.assertFalse(_is_whitespace('''A'''  )  )
        self.assertFalse(_is_whitespace('''-'''  )  )


    def      lowerCAmelCase_						(self  )     ->      Any:
        self.assertTrue(_is_control('''\u0005'''  )  )

        self.assertFalse(_is_control('''A'''  )  )
        self.assertFalse(_is_control(''' '''  )  )
        self.assertFalse(_is_control('''\t'''  )  )
        self.assertFalse(_is_control('''\r'''  )  )


    def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
        self.assertTrue(_is_punctuation('''-'''  )  )
        self.assertTrue(_is_punctuation('''$'''  )  )
        self.assertTrue(_is_punctuation('''`'''  )  )
        self.assertTrue(_is_punctuation('''.'''  )  )

        self.assertFalse(_is_punctuation('''A'''  )  )
        self.assertFalse(_is_punctuation(''' '''  )  )


    def      lowerCAmelCase_						(self  )     ->      str:
        __UpperCAmelCase									=			self.get_tokenizer()
        __UpperCAmelCase									=			self.get_rust_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase  ) for t in ['''Test''', '''\xad''', '''test''']]     ,      [['''[UNK]'''], [], ['''[UNK]''']]  )

        self.assertListEqual(
            [rust_tokenizer.tokenize(_lowerCAmelCase  ) for t in ['''Test''', '''\xad''', '''test''']]     ,      [['''[UNK]'''], [], ['''[UNK]''']]  )


    @slow
    def      lowerCAmelCase_						(self  )     ->      List[Any]:
        __UpperCAmelCase									=			self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''  )

        __UpperCAmelCase									=			tokenizer.encode('''sequence builders'''     ,      add_special_tokens=_lowerCAmelCase  )
        __UpperCAmelCase									=			tokenizer.encode('''multi-sequence build'''     ,      add_special_tokens=_lowerCAmelCase  )

        __UpperCAmelCase									=			tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase  )
        __UpperCAmelCase									=			tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase     ,      _lowerCAmelCase  )

        assert encoded_sentence == [101] + text + [102]
        assert encoded_pair == [101] + text + [102] + text_a + [102]


    def      lowerCAmelCase_						(self  )     ->      Any:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                __UpperCAmelCase									=			self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )

                __UpperCAmelCase									=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                __UpperCAmelCase									=			tokenizer_r.encode_plus(
                    _lowerCAmelCase     ,      return_attention_mask=_lowerCAmelCase     ,      return_token_type_ids=_lowerCAmelCase     ,      return_offsets_mapping=_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase     ,      )

                __UpperCAmelCase									=			tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase     ,      '''do_lower_case'''  ) else False
                __UpperCAmelCase									=			(
                    [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), '''A'''),
                        ((1, 2), ''','''),
                        ((3, 5), '''na'''),
                        ((5, 6), '''##ï'''),
                        ((6, 8), '''##ve'''),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), '''Allen'''),
                        ((21, 23), '''##NL'''),
                        ((23, 24), '''##P'''),
                        ((25, 33), '''sentence'''),
                        ((33, 34), '''.'''),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                    if not do_lower_case
                    else [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), '''a'''),
                        ((1, 2), ''','''),
                        ((3, 8), '''naive'''),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), '''allen'''),
                        ((21, 23), '''##nl'''),
                        ((23, 24), '''##p'''),
                        ((25, 33), '''sentence'''),
                        ((33, 34), '''.'''),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                )

                self.assertEqual(
                    [e[1] for e in expected_results]     ,      tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
                self.assertEqual([e[0] for e in expected_results]     ,      tokens['''offset_mapping''']  )






    def      lowerCAmelCase_						(self  )     ->      Any:
        __UpperCAmelCase									=			['''的''', '''人''', '''有''']
        __UpperCAmelCase									=			''''''.join(_lowerCAmelCase  )
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                __UpperCAmelCase									=			True
                __UpperCAmelCase									=			self.tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )
                __UpperCAmelCase									=			self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )

                __UpperCAmelCase									=			tokenizer_p.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
                __UpperCAmelCase									=			tokenizer_r.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )

                __UpperCAmelCase									=			tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
                __UpperCAmelCase									=			tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )

                # it is expected that each Chinese character is not preceded by "##"
                self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
                self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )

                __UpperCAmelCase									=			False
                __UpperCAmelCase									=			self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )
                __UpperCAmelCase									=			self.tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )

                __UpperCAmelCase									=			tokenizer_r.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
                __UpperCAmelCase									=			tokenizer_p.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )

                __UpperCAmelCase									=			tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
                __UpperCAmelCase									=			tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )

                # it is expected that only the first Chinese character is not preceded by "##".
                __UpperCAmelCase									=			[
                    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase  )
                ]
                self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
                self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )






import numpy as np


def       __a       (   SCREAMING_SNAKE_CASE      )      ->     Tuple:






    '''simple docstring'''
    __UpperCAmelCase					,     __UpperCAmelCase									=			a.shape
    __UpperCAmelCase									=			min(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )
    __UpperCAmelCase									=			np.eye(SCREAMING_SNAKE_CASE      )
    __UpperCAmelCase									=			a.copy()

    for k in range(t - 1      ):
        # select a column of modified matrix A':
        __UpperCAmelCase									=			r[k:, [k]]
        # construct first basis vector
        __UpperCAmelCase									=			np.zeros_like(SCREAMING_SNAKE_CASE      )
        __UpperCAmelCase									=			1.0
        # determine scaling factor
        __UpperCAmelCase									=			np.linalg.norm(SCREAMING_SNAKE_CASE      )
        # construct vector v for Householder reflection
        __UpperCAmelCase									=			x + np.sign(x[0]      ) * alpha * ea
        v /= np.linalg.norm(SCREAMING_SNAKE_CASE      )

        # construct the Householder matrix
        __UpperCAmelCase									=			np.eye(m - k      ) - 2.0 * v @ v.T
        # pad with ones and zeros as necessary
        __UpperCAmelCase									=			np.block([[np.eye(SCREAMING_SNAKE_CASE      ), np.zeros((k, m - k)      )], [np.zeros((m - k, k)      ), q_k]]      )

        __UpperCAmelCase									=			q @ q_k.T
        __UpperCAmelCase									=			q_k @ r

    return q, r


if __name__ == "__main__":
       import doctest

       doctest.testmod()




