





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class     SCREAMING_SNAKE_CASE__ (						UpperCamelCase__				,					unittest.TestCase      ):
 __SCREAMING_SNAKE_CASE							       =	BertTokenizer
 __SCREAMING_SNAKE_CASE							       =	BertTokenizerFast
 __SCREAMING_SNAKE_CASE							       =	True
 __SCREAMING_SNAKE_CASE							       =	True
 __SCREAMING_SNAKE_CASE							       =	filter_non_english



 def 				UpperCamelCase			(							self ):
        super().setUp()

        A__											=						[
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''[PAD]''',
            '''[MASK]''',
            '''want''',
            '''##want''',
            '''##ed''',
            '''wa''',
            '''un''',
            '''runn''',
            '''##ing''',
            ''',''',
            '''low''',
            '''lowest''',
        ]
        A__											=						os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['''vocab_file'''] )
        with open(self.vocab_file,'''w''',encoding='''utf-8''' ) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )



 def 				UpperCamelCase			(							self,__lowerCamelCase ):
        A__											=						'''UNwant\u00E9d,running'''
        A__											=						'''unwanted, running'''
        return input_text, output_text



 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer_class(self.vocab_file )

        A__											=						tokenizer.tokenize('''UNwant\u00E9d,running''' )
        self.assertListEqual(__lowerCamelCase,['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''] )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase ),[9, 6, 7, 12, 10, 11] )



 def 				UpperCamelCase			(							self ):
        if not self.test_rust_tokenizer:
               return

        A__											=						self.get_tokenizer()
        A__											=						self.get_rust_tokenizer()

        A__											=						'''UNwant\u00E9d,running'''

        A__											=						tokenizer.tokenize(__lowerCamelCase )
        A__											=						rust_tokenizer.tokenize(__lowerCamelCase )
        self.assertListEqual(__lowerCamelCase,__lowerCamelCase )

        A__											=						tokenizer.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase )
        A__											=						rust_tokenizer.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase )
        self.assertListEqual(__lowerCamelCase,__lowerCamelCase )

        A__											=						self.get_rust_tokenizer()
        A__											=						tokenizer.encode(__lowerCamelCase )
        A__											=						rust_tokenizer.encode(__lowerCamelCase )
        self.assertListEqual(__lowerCamelCase,__lowerCamelCase )

        # With lower casing
        A__											=						self.get_tokenizer(do_lower_case=__lowerCamelCase )
        A__											=						self.get_rust_tokenizer(do_lower_case=__lowerCamelCase )

        A__											=						'''UNwant\u00E9d,running'''

        A__											=						tokenizer.tokenize(__lowerCamelCase )
        A__											=						rust_tokenizer.tokenize(__lowerCamelCase )
        self.assertListEqual(__lowerCamelCase,__lowerCamelCase )

        A__											=						tokenizer.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase )
        A__											=						rust_tokenizer.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase )
        self.assertListEqual(__lowerCamelCase,__lowerCamelCase )

        A__											=						self.get_rust_tokenizer()
        A__											=						tokenizer.encode(__lowerCamelCase )
        A__											=						rust_tokenizer.encode(__lowerCamelCase )
        self.assertListEqual(__lowerCamelCase,__lowerCamelCase )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''' ),['''ah''', '''\u535A''', '''\u63A8''', '''zz'''] )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer(do_lower_case=__lowerCamelCase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' ),['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' ),['''hello'''] )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer(do_lower_case=__lowerCamelCase,strip_accents=__lowerCamelCase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' ),['''h\u00E9llo'''] )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer(do_lower_case=__lowerCamelCase,strip_accents=__lowerCamelCase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' ),['''hello'''] )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer(do_lower_case=__lowerCamelCase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' ),['''hello'''] )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer(do_lower_case=__lowerCamelCase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' ),['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer(do_lower_case=__lowerCamelCase,strip_accents=__lowerCamelCase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer(do_lower_case=__lowerCamelCase,strip_accents=__lowerCamelCase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer(do_lower_case=__lowerCamelCase,never_split=['''[UNK]'''] )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''' ),['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''] )



 def 				UpperCamelCase			(							self ):
        A__											=						BasicTokenizer()
        A__											=						'''a\n\'ll !!to?\'d of, can\'t.'''
        A__											=						['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
        self.assertListEqual(tokenizer.tokenize(__lowerCamelCase ),__lowerCamelCase )



 def 				UpperCamelCase			(							self ):
        A__											=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

        A__											=						{}
        for i, token in enumerate(__lowerCamelCase ):
               A__											=						i
        A__											=						WordpieceTokenizer(vocab=__lowerCamelCase,unk_token='''[UNK]''' )

        self.assertListEqual(tokenizer.tokenize('''''' ),[] )

        self.assertListEqual(tokenizer.tokenize('''unwanted running''' ),['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''] )

        self.assertListEqual(tokenizer.tokenize('''unwantedX running''' ),['''[UNK]''', '''runn''', '''##ing'''] )



 def 				UpperCamelCase			(							self ):
        self.assertTrue(_is_whitespace(''' ''' ) )
        self.assertTrue(_is_whitespace('''\t''' ) )
        self.assertTrue(_is_whitespace('''\r''' ) )
        self.assertTrue(_is_whitespace('''\n''' ) )
        self.assertTrue(_is_whitespace('''\u00A0''' ) )

        self.assertFalse(_is_whitespace('''A''' ) )
        self.assertFalse(_is_whitespace('''-''' ) )



 def 				UpperCamelCase			(							self ):
        self.assertTrue(_is_control('''\u0005''' ) )

        self.assertFalse(_is_control('''A''' ) )
        self.assertFalse(_is_control(''' ''' ) )
        self.assertFalse(_is_control('''\t''' ) )
        self.assertFalse(_is_control('''\r''' ) )



 def 				UpperCamelCase			(							self ):
        self.assertTrue(_is_punctuation('''-''' ) )
        self.assertTrue(_is_punctuation('''$''' ) )
        self.assertTrue(_is_punctuation('''`''' ) )
        self.assertTrue(_is_punctuation('''.''' ) )

        self.assertFalse(_is_punctuation('''A''' ) )
        self.assertFalse(_is_punctuation(''' ''' ) )



 def 				UpperCamelCase			(							self ):
        A__											=						self.get_tokenizer()
        A__											=						self.get_rust_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(__lowerCamelCase ) for t in ['''Test''', '''\xad''', '''test''']],[['''[UNK]'''], [], ['''[UNK]''']] )

        self.assertListEqual(
            [rust_tokenizer.tokenize(__lowerCamelCase ) for t in ['''Test''', '''\xad''', '''test''']],[['''[UNK]'''], [], ['''[UNK]''']] )



 @slow
 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer_class.from_pretrained('''bert-base-uncased''' )

        A__											=						tokenizer.encode('''sequence builders''',add_special_tokens=__lowerCamelCase )
        A__											=						tokenizer.encode('''multi-sequence build''',add_special_tokens=__lowerCamelCase )

        A__											=						tokenizer.build_inputs_with_special_tokens(__lowerCamelCase )
        A__											=						tokenizer.build_inputs_with_special_tokens(__lowerCamelCase,__lowerCamelCase )

        assert encoded_sentence == [101] + text + [102]
        assert encoded_pair == [101] + text + [102] + text_a + [102]



 def 				UpperCamelCase			(							self ):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
                      A__											=						self.rust_tokenizer_class.from_pretrained(__lowerCamelCase,**__lowerCamelCase )

                      A__											=						f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                      A__											=						tokenizer_r.encode_plus(
                          __lowerCamelCase,return_attention_mask=__lowerCamelCase,return_token_type_ids=__lowerCamelCase,return_offsets_mapping=__lowerCamelCase,add_special_tokens=__lowerCamelCase,)

                      A__											=						tokenizer_r.do_lower_case if hasattr(__lowerCamelCase,'''do_lower_case''' ) else False
                      A__											=						(
                          [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), '''A'''),
                              ((1, 2), ''','''),
                              ((3, 5), '''na'''),
                              ((5, 6), '''##ï'''),
                              ((6, 8), '''##ve'''),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), '''Allen'''),
                              ((21, 23), '''##NL'''),
                              ((23, 24), '''##P'''),
                              ((25, 33), '''sentence'''),
                              ((33, 34), '''.'''),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                          if not do_lower_case
                          else [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), '''a'''),
                              ((1, 2), ''','''),
                              ((3, 8), '''naive'''),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), '''allen'''),
                              ((21, 23), '''##nl'''),
                              ((23, 24), '''##p'''),
                              ((25, 33), '''sentence'''),
                              ((33, 34), '''.'''),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                      )

                      self.assertEqual(
                          [e[1] for e in expected_results],tokenizer_r.convert_ids_to_tokens(tokens['''input_ids'''] ) )
                      self.assertEqual([e[0] for e in expected_results],tokens['''offset_mapping'''] )




 def 				UpperCamelCase			(							self ):
        A__											=						['''的''', '''人''', '''有''']
        A__											=						''''''.join(__lowerCamelCase )
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
                      A__											=						True
                      A__											=						self.tokenizer_class.from_pretrained(__lowerCamelCase,**__lowerCamelCase )
                      A__											=						self.rust_tokenizer_class.from_pretrained(__lowerCamelCase,**__lowerCamelCase )

                      A__											=						tokenizer_p.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase )
                      A__											=						tokenizer_r.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase )

                      A__											=						tokenizer_r.convert_ids_to_tokens(__lowerCamelCase )
                      A__											=						tokenizer_p.convert_ids_to_tokens(__lowerCamelCase )

                      # it is expected that each Chinese character is not preceded by "##"
                      self.assertListEqual(__lowerCamelCase,__lowerCamelCase )
                      self.assertListEqual(__lowerCamelCase,__lowerCamelCase )

                      A__											=						False
                      A__											=						self.rust_tokenizer_class.from_pretrained(__lowerCamelCase,**__lowerCamelCase )
                      A__											=						self.tokenizer_class.from_pretrained(__lowerCamelCase,**__lowerCamelCase )

                      A__											=						tokenizer_r.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase )
                      A__											=						tokenizer_p.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase )

                      A__											=						tokenizer_r.convert_ids_to_tokens(__lowerCamelCase )
                      A__											=						tokenizer_p.convert_ids_to_tokens(__lowerCamelCase )

                      # it is expected that only the first Chinese character is not preceded by "##".
                      A__											=						[
                          f"##{token}" if idx != 0 else token for idx, token in enumerate(__lowerCamelCase )
                      ]
                      self.assertListEqual(__lowerCamelCase,__lowerCamelCase )
                      self.assertListEqual(__lowerCamelCase,__lowerCamelCase )












import unittest

from transformers import AutoConfig, AutoTokenizer, BertConfig, TensorType, is_flax_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, slow


if is_flax_available():
       import jax

       from transformers.models.auto.modeling_flax_auto import FlaxAutoModel
       from transformers.models.bert.modeling_flax_bert import FlaxBertModel
       from transformers.models.roberta.modeling_flax_roberta import FlaxRobertaModel




@require_flax
class     SCREAMING_SNAKE_CASE__ (						unittest.TestCase      ):



 @slow
 def 				UpperCamelCase			(							self ):
        for model_name in ["bert-base-cased", "bert-large-uncased"]:
               with self.subTest(__lowerCamelCase ):
                      A__											=						AutoConfig.from_pretrained(__lowerCamelCase )
                      self.assertIsNotNone(__lowerCamelCase )
                      self.assertIsInstance(__lowerCamelCase,__lowerCamelCase )

                      A__											=						FlaxAutoModel.from_pretrained(__lowerCamelCase )
                      self.assertIsNotNone(__lowerCamelCase )
                      self.assertIsInstance(__lowerCamelCase,__lowerCamelCase )



 @slow
 def 				UpperCamelCase			(							self ):
        for model_name in ["roberta-base", "roberta-large"]:
               with self.subTest(__lowerCamelCase ):
                      A__											=						AutoConfig.from_pretrained(__lowerCamelCase )
                      self.assertIsNotNone(__lowerCamelCase )
                      self.assertIsInstance(__lowerCamelCase,__lowerCamelCase )

                      A__											=						FlaxAutoModel.from_pretrained(__lowerCamelCase )
                      self.assertIsNotNone(__lowerCamelCase )
                      self.assertIsInstance(__lowerCamelCase,__lowerCamelCase )



 @slow
 def 				UpperCamelCase			(							self ):
        for model_name in ["bert-base-cased", "bert-large-uncased"]:
               A__											=						AutoTokenizer.from_pretrained(__lowerCamelCase )
               A__											=						FlaxBertModel.from_pretrained(__lowerCamelCase )
               A__											=						tokenizer('''Do you support jax jitted function?''',return_tensors=TensorType.JAX )

               @jax.jit
               def eval(**__lowerCamelCase ):
                      return model(**__lowerCamelCase )

               eval(**__lowerCamelCase ).block_until_ready()



 @slow
 def 				UpperCamelCase			(							self ):
        for model_name in ["roberta-base", "roberta-large"]:
               A__											=						AutoTokenizer.from_pretrained(__lowerCamelCase )
               A__											=						FlaxRobertaModel.from_pretrained(__lowerCamelCase )
               A__											=						tokenizer('''Do you support jax jitted function?''',return_tensors=TensorType.JAX )

               @jax.jit
               def eval(**__lowerCamelCase ):
                      return model(**__lowerCamelCase )

               eval(**__lowerCamelCase ).block_until_ready()



 def 				UpperCamelCase			(							self ):
        with self.assertRaisesRegex(
            __lowerCamelCase,'''bert-base is not a local folder and is not a valid model identifier''' ):
               A__											=						FlaxAutoModel.from_pretrained('''bert-base''' )



 def 				UpperCamelCase			(							self ):
        with self.assertRaisesRegex(
            __lowerCamelCase,r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
               A__											=						FlaxAutoModel.from_pretrained(__lowerCamelCase,revision='''aaaaaa''' )



 def 				UpperCamelCase			(							self ):
        with self.assertRaisesRegex(
            __lowerCamelCase,'''hf-internal-testing/config-no-model does not appear to have a file named flax_model.msgpack''',):
               A__											=						FlaxAutoModel.from_pretrained('''hf-internal-testing/config-no-model''' )




 def 				UpperCamelCase			(							self ):
        with self.assertRaisesRegex(__lowerCamelCase,'''Use `from_pt=True` to load this model''' ):
               A__											=						FlaxAutoModel.from_pretrained('''hf-internal-testing/tiny-bert-pt-only''' )





