


import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


_A		       =						get_tests_dir('fixtures/test_sentencepiece.model')
_A		       =						get_tests_dir('fixtures/test_sentencepiece_bpe.model')

_A		       =						'pt' if is_torch_available() else 'tf'






@require_sentencepiece
@require_tokenizers
class     UpperCAmelCase__       (					A_   ,  unittest.TestCase		):



    """simple docstring"""






    UpperCAmelCase__						:		Tuple      =  CamembertTokenizer
    UpperCAmelCase__						:		Tuple      =  CamembertTokenizerFast
    UpperCAmelCase__						:		Optional[Any]      =  True
    UpperCAmelCase__						:		List[Any]      =  True






    def _a					(			self    )  ->       List[str]:
         super().setUp()

         # We have a SentencePiece fixture for testing
         __UpperCamelCase    							=CamembertTokenizer(A_    )
         tokenizer.save_pretrained(self.tmpdirname    )






    def _a					(			self    )  ->       str:
         __UpperCamelCase    							='<pad>'
         __UpperCamelCase    							=1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_    )					,   A_    )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_    )					,   A_    )






    def _a					(			self    )  ->       int:
         __UpperCamelCase    							=list(self.get_tokenizer().get_vocab().keys()    )

         self.assertEqual(vocab_keys[0]					,   '<s>NOTUSED'    )
         self.assertEqual(vocab_keys[1]					,   '<pad>'    )
         self.assertEqual(vocab_keys[-1]					,   '<mask>'    )
         self.assertEqual(len(A_    )					,   1004    )






    def _a					(			self    )  ->       int:
         self.assertEqual(self.get_tokenizer().vocab_size					,   1005    )






    def _a					(			self    )  ->       Optional[int]:
         __UpperCamelCase    							=CamembertTokenizer(A_    )
         tokenizer.save_pretrained(self.tmpdirname    )
         __UpperCamelCase    							=CamembertTokenizerFast.from_pretrained(self.tmpdirname    )

         __UpperCamelCase    							='I was born in 92000, and this is falsé.'

         __UpperCamelCase    							=tokenizer.encode(A_    )
         __UpperCamelCase    							=rust_tokenizer.encode(A_    )
         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=tokenizer.encode(A_					,   add_special_tokens=A_    )
         __UpperCamelCase    							=rust_tokenizer.encode(A_					,   add_special_tokens=A_    )
         self.assertListEqual(A_					,   A_    )

         # <unk> tokens are not the same for `rust` than for `slow`.
         # Because spm gives back raw token instead of `unk` in EncodeAsPieces
         # tokens = tokenizer.tokenize(sequence)
         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(A_    )
         __UpperCamelCase    							=rust_tokenizer.tokenize(A_    )
         self.assertListEqual(A_					,   A_    )






    def _a					(			self    )  ->       Optional[Any]:
         if not self.test_rust_tokenizer:
              return

         __UpperCamelCase    							=self.get_tokenizer()
         __UpperCamelCase    							=self.get_rust_tokenizer()

         __UpperCamelCase    							='I was born in 92000, and this is falsé.'

         __UpperCamelCase    							=tokenizer.tokenize(A_    )
         __UpperCamelCase    							=rust_tokenizer.tokenize(A_    )
         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=tokenizer.encode(A_					,   add_special_tokens=A_    )
         __UpperCamelCase    							=rust_tokenizer.encode(A_					,   add_special_tokens=A_    )
         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=self.get_rust_tokenizer()
         __UpperCamelCase    							=tokenizer.encode(A_    )
         __UpperCamelCase    							=rust_tokenizer.encode(A_    )
         self.assertListEqual(A_					,   A_    )






    @slow
    def _a					(			self    )  ->       Union[str, Any]:
         # fmt: off
         __UpperCamelCase    							={'input_ids': [[5, 54, 7196, 297, 30, 23, 776, 18, 11, 3215, 3705, 8252, 22, 3164, 1181, 2116, 29, 16, 813, 25, 791, 3314, 20, 3446, 38, 27575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9088, 20, 1517, 8, 22804, 18818, 10, 38, 629, 607, 607, 142, 19, 7196, 867, 56, 10326, 24, 2267, 20, 416, 5072, 15612, 233, 734, 7, 2399, 27, 16, 3015, 1649, 7, 24, 20, 4338, 2399, 27, 13, 3400, 14, 13, 6189, 8, 930, 9, 6]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         # camembert is a french model. So we also use french texts.
         __UpperCamelCase    							=[
             'Le transformeur est un modèle d\'apprentissage profond introduit en 2017, '
             'utilisé principalement dans le domaine du traitement automatique des langues (TAL).',
             'À l\'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus '
             'pour gérer des données séquentielles, telles que le langage naturel, pour des tâches '
             'telles que la traduction et la synthèse de texte.',
         ]

         self.tokenizer_integration_test_util(
             expected_encoding=A_					,   model_name='camembert-base'					,   revision='3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf'					,   sequences=A_					,   )









'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowerCamelCase__					=   logging.get_logger(__name__)

lowerCamelCase__					=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase__					=   {
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

lowerCamelCase__					=   {
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

lowerCamelCase__					=   {
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}






class 		lowerCAmelCase__	(	UpperCAmelCase__		):
			lowerCAmelCase       :		Union[str, Any]				      =   VOCAB_FILES_NAMES
			lowerCAmelCase       :		Dict				      =   PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase       :		Tuple				      =   PRETRAINED_INIT_CONFIGURATION
			lowerCAmelCase       :		Optional[int]				      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase       :		Any				      =   BertTokenizer




			def __init__(						self     :   int		,     lowerCamelCase__     :   Optional[Any]=None		,     lowerCamelCase__     :   Optional[int]=None		,     lowerCamelCase__     :   str=True		,     lowerCamelCase__     :   Tuple="[UNK]"		,     lowerCamelCase__     :   str="[SEP]"		,     lowerCamelCase__     :   Optional[Any]="[PAD]"		,     lowerCamelCase__     :   List[str]="[CLS]"		,     lowerCamelCase__     :   Union[str, Any]="[MASK]"		,     lowerCamelCase__     :   str=True		,     lowerCamelCase__     :   Dict=None		,     **lowerCamelCase__     :   Union[str, Any]		,     )      ->Tuple:



					'''simple docstring'''







					super().__init__(
					    lowerCamelCase__		,     tokenizer_file=lowerCamelCase__		,     do_lower_case=lowerCamelCase__		,     unk_token=lowerCamelCase__		,     sep_token=lowerCamelCase__		,     pad_token=lowerCamelCase__		,     cls_token=lowerCamelCase__		,     mask_token=lowerCamelCase__		,     tokenize_chinese_chars=lowerCamelCase__		,     strip_accents=lowerCamelCase__		,     **lowerCamelCase__		,     )

					_UpperCAmelCase						:				Optional[Any]	    =					json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
					if (
					    normalizer_state.get("lowercase"		,     lowerCamelCase__       ) != do_lower_case
					    or normalizer_state.get("strip_accents"		,     lowerCamelCase__       ) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"		,     lowerCamelCase__       ) != tokenize_chinese_chars
					):
							_UpperCAmelCase						:				str	    =					getattr(lowerCamelCase__		,     normalizer_state.pop("type"       )       )
							_UpperCAmelCase						:				Optional[Any]	    =					do_lower_case
							_UpperCAmelCase						:				Any	    =					strip_accents
							_UpperCAmelCase						:				List[Any]	    =					tokenize_chinese_chars
							_UpperCAmelCase						:				int	    =					normalizer_class(**lowerCamelCase__       )

					_UpperCAmelCase						:				Union[str, Any]	    =					do_lower_case




			def 							lowerCAmelCase__			(						self     :   List[str]		,     lowerCamelCase__     :   List[str]		,     lowerCamelCase__     :   str=None       )      ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

					return output




			def 							lowerCAmelCase__			(						self     :   List[Any]		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None       )      ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Tuple	    =					[self.sep_token_id]
					_UpperCAmelCase						:				int	    =					[self.cls_token_id]
					if token_ids_a is None:
							return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






			def 							lowerCAmelCase__			(						self     :   List[Any]		,     lowerCamelCase__     :   str		,     lowerCamelCase__     :   Optional[str] = None       )      ->Tuple[str]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					self._tokenizer.model.save(lowerCamelCase__		,     name=lowerCamelCase__       )
					return tuple(lowerCamelCase__       )



