


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase											=       get_tests_dir('fixtures/spiece.model')




@require_sentencepiece
@require_tokenizers
class A  (   UpperCamelCase_    ,							unittest.TestCase     ):
      UpperCamelCase__							:  Union[str, Any]      =AlbertTokenizer
      UpperCamelCase__							:  Any      =AlbertTokenizerFast
      UpperCamelCase__							:  Union[str, Any]      =True
      UpperCamelCase__							:  Optional[Any]      =True
      UpperCamelCase__							:  Dict      =True






      def        lowerCamelCase       (					self						:     int )					->						Optional[Any]:
             """simple docstring"""




             super().setUp()

             # We have a SentencePiece fixture for testing
             _lowerCamelCase       :		List[Any]		=AlbertTokenizer(lowercase_ )
             tokenizer.save_pretrained(self.tmpdirname )






      def        lowerCamelCase       (					self						:     List[Any]			,   lowercase_						:     Optional[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		='this is a test'
             _lowerCamelCase       :		str		='this is a test'
             return input_text, output_text






      def        lowerCamelCase       (					self						:     List[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		='<pad>'
             _lowerCamelCase       :		Optional[Any]		=0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_ )			,   lowercase_ )
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_ )			,   lowercase_ )






      def        lowerCamelCase       (					self						:     int )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=list(self.get_tokenizer().get_vocab().keys() )

             self.assertEqual(vocab_keys[0]			,   '<pad>' )
             self.assertEqual(vocab_keys[1]			,   '<unk>' )
             self.assertEqual(vocab_keys[-1]			,   '▁eloquent' )
             self.assertEqual(len(lowercase_ )			,   3_0000 )






      def        lowerCamelCase       (					self						:     List[Any] )					->						List[str]:
             """simple docstring"""




             self.assertEqual(self.get_tokenizer().vocab_size			,   3_0000 )






      def        lowerCamelCase       (					self						:     str )					->						List[str]:
             """simple docstring"""




             if not self.test_rust_tokenizer:
                    return

             _lowerCamelCase       :		Optional[int]		=self.get_tokenizer()
             _lowerCamelCase       :		Dict		=self.get_rust_tokenizer()

             _lowerCamelCase       :		int		='I was born in 92000, and this is falsé.'

             _lowerCamelCase       :		str		=tokenizer.tokenize(lowercase_ )
             _lowerCamelCase       :		Optional[int]		=rust_tokenizer.tokenize(lowercase_ )
             self.assertListEqual(lowercase_			,   lowercase_ )

             _lowerCamelCase       :		List[str]		=tokenizer.encode(lowercase_			,   add_special_tokens=lowercase_ )
             _lowerCamelCase       :		Optional[Any]		=rust_tokenizer.encode(lowercase_			,   add_special_tokens=lowercase_ )
             self.assertListEqual(lowercase_			,   lowercase_ )

             _lowerCamelCase       :		Optional[int]		=self.get_rust_tokenizer()
             _lowerCamelCase       :		Tuple		=tokenizer.encode(lowercase_ )
             _lowerCamelCase       :		List[str]		=rust_tokenizer.encode(lowercase_ )
             self.assertListEqual(lowercase_			,   lowercase_ )






      def        lowerCamelCase       (					self						:     int )					->						List[Any]:
             """simple docstring"""




             _lowerCamelCase       :		str		=AlbertTokenizer(lowercase_			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Union[str, Any]		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁this', '▁is', '▁a', '▁test'] )

             self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_ )			,   [48, 25, 21, 1289] )

             _lowerCamelCase       :		Dict		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.'] )
             _lowerCamelCase       :		Tuple		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(lowercase_			,   [31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9] )

             _lowerCamelCase       :		Optional[int]		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']			,   )






      def        lowerCamelCase       (					self						:     str )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		Optional[int]		=AlbertTokenizer(lowercase_ )

             _lowerCamelCase       :		Any		=tokenizer.encode('sequence builders' )
             _lowerCamelCase       :		Any		=tokenizer.encode('multi-sequence build' )

             _lowerCamelCase       :		Dict		=tokenizer.build_inputs_with_special_tokens(lowercase_ )
             _lowerCamelCase       :		List[str]		=tokenizer.build_inputs_with_special_tokens(lowercase_			,   lowercase_ )

             assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
             assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
                 tokenizer.sep_token_id
             ]






      @slow
      def        lowerCamelCase       (					self						:     int )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		int		={'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'input_ids': [[2, 2_1970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 1_2051, 18, 17, 7103, 2153, 673, 8, 3515, 1_8684, 8, 4461, 6, 1927, 297, 8, 1_2060, 2607, 18, 13, 5, 4461, 15, 1_0538, 38, 8, 135, 15, 822, 58, 15, 993, 1_0363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 1_0641, 6, 29, 84, 2512, 2430, 782, 1_8684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 1_1712, 15, 7103, 2153, 673, 17, 2_4883, 9990, 9, 3], [2, 1_1502, 25, 1006, 20, 782, 8, 1_1809, 855, 1732, 1_9393, 1_8667, 37, 367, 2_1018, 69, 1854, 34, 1_1860, 1_9124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 1_7659, 84, 14, 1_6792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=lowercase_			,   model_name='albert-base-v2'			,   revision='6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'			,   )






import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


lowerCamelCase											=       logging.getLogger()




@unittest.skip('Temporarily disable the doc tests.'     )
@require_torch
@require_tf
@slow
class A  (   unittest.TestCase     ):






      def        lowerCamelCase       (					self						:     Dict			,   lowercase_						:     Path			,   lowercase_						:     Union[str, None] = None			,   lowercase_						:     Union[List[str], None] = None			,   lowercase_						:     Union[str, List[str], None] = None			,   lowercase_						:     bool = True			,   )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=[file for file in os.listdir(lowercase_ ) if os.path.isfile(os.path.join(lowercase_			,   lowercase_ ) )]

             if identifier is not None:
                    _lowerCamelCase       :		Any		=[file for file in files if identifier in file]

             if n_identifier is not None:
                    if isinstance(lowercase_			,   lowercase_ ):
                           for n_ in n_identifier:
                                  _lowerCamelCase       :		Any		=[file for file in files if n_ not in file]
                    else:
                           _lowerCamelCase       :		List[Any]		=[file for file in files if n_identifier not in file]

             _lowerCamelCase       :		List[Any]		=ignore_files or []
             ignore_files.append('__init__.py' )
             _lowerCamelCase       :		List[Any]		=[file for file in files if file not in ignore_files]

             for file in files:
                    # Open all files
                    print('Testing'			,   lowercase_ )

                    if only_modules:
                           _lowerCamelCase       :		Optional[int]		=file.split('.' )[0]
                           try:
                                  _lowerCamelCase       :		Any		=getattr(lowercase_			,   lowercase_ )
                                  _lowerCamelCase       :		Optional[int]		=doctest.DocTestSuite(lowercase_ )
                                  _lowerCamelCase       :		List[Any]		=unittest.TextTestRunner().run(lowercase_ )
                                  self.assertIs(len(result.failures )			,   0 )
                           except AttributeError:
                                  logger.info(F'''{module_identifier} is not a module.''' )
                    else:
                           _lowerCamelCase       :		int		=doctest.testfile(str('..' / directory / file )			,   optionflags=doctest.ELLIPSIS )
                           self.assertIs(result.failed			,   0 )






      def        lowerCamelCase       (					self						:     Any )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		int		=Path('src/transformers' )
             _lowerCamelCase       :		Optional[Any]		='modeling'
             _lowerCamelCase       :		List[str]		=[
                 'modeling_ctrl.py',
                 'modeling_tf_ctrl.py',
             ]
             self.analyze_directory(lowercase_			,   identifier=lowercase_			,   ignore_files=lowercase_ )






      def        lowerCamelCase       (					self						:     List[str] )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		Optional[int]		=Path('src/transformers' )
             _lowerCamelCase       :		Union[str, Any]		='tokenization'
             self.analyze_directory(lowercase_			,   identifier=lowercase_ )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		str		=Path('src/transformers' )
             _lowerCamelCase       :		Any		='configuration'
             self.analyze_directory(lowercase_			,   identifier=lowercase_ )






      def        lowerCamelCase       (					self						:     List[str] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Union[str, Any]		=Path('src/transformers' )
             _lowerCamelCase       :		int		=['configuration', 'modeling', 'tokenization']
             self.analyze_directory(lowercase_			,   n_identifier=lowercase_ )






      def        lowerCamelCase       (					self						:     Optional[Any] )					->						int:
             """simple docstring"""




             _lowerCamelCase       :		str		=Path('docs/source' )
             _lowerCamelCase       :		Dict		=['favicon.ico']
             self.analyze_directory(lowercase_			,   ignore_files=lowercase_			,   only_modules=lowercase_ )


