

import logging
import os
import threading
import time


try:
	import warnings
except ImportError:
	A_       : Tuple         =			None

try:
	import msvcrt
except ImportError:
	A_       : List[str]         =			None

try:
	import fcntl
except ImportError:
	A_       : Any         =			None


# Backward compatibility
# ------------------------------------------------
try:
	TimeoutError
except NameError:
	A_       : Optional[int]         =			OSError


# Data
# ------------------------------------------------
A_       : int         =			[
    '''Timeout''',
    '''BaseFileLock''',
    '''WindowsFileLock''',
    '''UnixFileLock''',
    '''SoftFileLock''',
    '''FileLock''',
]

A_       : Optional[int]         =			'''3.0.12'''


A_       : Dict         =			None

def 						UpperCamelCase					()     ->      Optional[int]:
	global _logger
	A__							:    Union[str, Any]        =		_logger or logging.getLogger(__name__					)
	return _logger



class     _a						(__snake_case				):


					'''simple docstring'''






					def __init__( self    ,       A__			):
						A__							:    Dict        =		lock_file
						return None






					def __str__( self			):
						A__							:    str        =		F"""The file lock \'{self.lock_file}\' could not be acquired."""
						return temp



class     _a						:


					'''simple docstring'''






					def __init__( self    ,       A__			):
						A__							:    int        =		lock
						return None






					def __enter__( self			):
						return self.lock






					def __exit__( self    ,       A__    ,       A__    ,       A__			):
						self.lock.release()
						return None



class     _a						:


					'''simple docstring'''






					def __init__( self    ,       A__    ,       A__=-1    ,       A__=None			):
						A__							:    Any        =		max_filename_length if max_filename_length is not None else 255
						# Hash the filename if it's too long
						A__							:    int        =		self.hash_filename_if_too_long(__SCREAMING_SNAKE_CASE    ,       __SCREAMING_SNAKE_CASE			)
						# The path to the lock file.
						A__							:    Union[str, Any]        =		lock_file

						# The file descriptor for the *_lock_file* as it is returned by the
						# os.open() function.
						# This file lock is only NOT None, if the object currently holds the
						# lock.
						A__							:    Union[str, Any]        =		None

						# The default timeout value.
						A__							:    Optional[Any]        =		timeout

						# We use this lock primarily for the lock counter.
						A__							:    Optional[int]        =		threading.Lock()

						# The lock counter is used for implementing the nested locking
						# mechanism. Whenever the lock is acquired, the counter is increased and
						# the lock is only released, when this value is 0 again.
						A__							:    List[Any]        =		0
						return None






					@property
					def   __A			( self			):
						return self._lock_file






					@property
					def   __A			( self			):
						return self._timeout






					@timeout.setter
					def   __A			( self    ,       A__			):
						A__							:    str        =		float(__SCREAMING_SNAKE_CASE			)
						return None






					def   __A			( self			):
						raise NotImplementedError()






					def   __A			( self			):
						raise NotImplementedError()






					@property
					def   __A			( self			):
						return self._lock_file_fd is not None






					def   __A			( self    ,       A__=None    ,       A__=0.0_5			):
						# Use the default timeout, if no timeout is provided.
						if timeout is None:
							A__							:    Optional[int]        =		self.timeout

						# Increment the number right at the beginning.
						# We can still undo it, if something fails.
						with self._thread_lock:
							self._lock_counter += 1

						A__							:    List[Any]        =		id(self			)
						A__							:    Any        =		self._lock_file
						A__							:    int        =		time.time()
						try:
							while True:
								with self._thread_lock:
									if not self.is_locked:
										logger().debug(F"""Attempting to acquire lock {lock_id} on {lock_filename}"""			)
										self._acquire()

								if self.is_locked:
									logger().debug(F"""Lock {lock_id} acquired on {lock_filename}"""			)
									break
								elif timeout >= 0 and time.time() - start_time > timeout:
									logger().debug(F"""Timeout on acquiring lock {lock_id} on {lock_filename}"""			)
									raise Timeout(self._lock_file			)
								else:
									logger().debug(
									    F"""Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ..."""			)
									time.sleep(__SCREAMING_SNAKE_CASE			)
						except:  # noqa
							# Something did go wrong, so decrement the counter.
							with self._thread_lock:
								A__							:    Any        =		max(0    ,       self._lock_counter - 1			)

							raise
						return _Acquire_ReturnProxy(lock=self			)






					def   __A			( self    ,       A__=False			):
						with self._thread_lock:
							if self.is_locked:
								self._lock_counter -= 1

								if self._lock_counter == 0 or force:
									A__							:    List[str]        =		id(self			)
									A__							:    Union[str, Any]        =		self._lock_file

									logger().debug(F"""Attempting to release lock {lock_id} on {lock_filename}"""			)
									self._release()
									A__							:    Union[str, Any]        =		0
									logger().debug(F"""Lock {lock_id} released on {lock_filename}"""			)

						return None






					def __enter__( self			):
						self.acquire()
						return self






					def __exit__( self    ,       A__    ,       A__    ,       A__			):
						self.release()
						return None






					def __del__( self			):
						self.release(force=__SCREAMING_SNAKE_CASE			)
						return None






					def   __A			( self    ,       A__    ,       A__			):
						A__							:    Union[str, Any]        =		os.path.basename(__SCREAMING_SNAKE_CASE			)
						if len(__SCREAMING_SNAKE_CASE			) > max_length and max_length > 0:
							A__							:    Optional[Any]        =		os.path.dirname(__SCREAMING_SNAKE_CASE			)
							A__							:    Dict        =		str(hash(__SCREAMING_SNAKE_CASE			)			)
							A__							:    int        =		filename[: max_length - len(__SCREAMING_SNAKE_CASE			) - 8] + """...""" + hashed_filename + """.lock"""
							return os.path.join(__SCREAMING_SNAKE_CASE    ,       __SCREAMING_SNAKE_CASE			)
						else:
							return path



class     _a						(__snake_case				):


					'''simple docstring'''






					def __init__( self    ,       A__    ,       A__=-1    ,       A__=None			):
						from .file_utils import relative_to_absolute_path

						super().__init__(__SCREAMING_SNAKE_CASE    ,       timeout=__SCREAMING_SNAKE_CASE    ,       max_filename_length=__SCREAMING_SNAKE_CASE			)
						A__							:    List[str]        =		"""\\\\?\\""" + relative_to_absolute_path(self.lock_file			)






					def   __A			( self			):
						A__							:    List[str]        =		os.O_RDWR | os.O_CREAT | os.O_TRUNC

						try:
							A__							:    Optional[Any]        =		os.open(self._lock_file    ,       __SCREAMING_SNAKE_CASE			)
						except OSError:
							pass
						else:
							try:
								msvcrt.locking(__SCREAMING_SNAKE_CASE    ,       msvcrt.LK_NBLCK    ,       1			)
							except OSError:
								os.close(__SCREAMING_SNAKE_CASE			)
							else:
								A__							:    Tuple        =		fd
						return None






					def   __A			( self			):
						A__							:    Tuple        =		self._lock_file_fd
						A__							:    Tuple        =		None
						msvcrt.locking(__SCREAMING_SNAKE_CASE    ,       msvcrt.LK_UNLCK    ,       1			)
						os.close(__SCREAMING_SNAKE_CASE			)

						try:
							os.remove(self._lock_file			)
						# Probably another instance of the application
						# that acquired the file lock.
						except OSError:
							pass
						return None



class     _a						(__snake_case				):


					'''simple docstring'''






					def __init__( self    ,       A__    ,       A__=-1    ,       A__=None			):
						A__							:    str        =		os.statvfs(os.path.dirname(__SCREAMING_SNAKE_CASE			)			).f_namemax
						super().__init__(__SCREAMING_SNAKE_CASE    ,       timeout=__SCREAMING_SNAKE_CASE    ,       max_filename_length=__SCREAMING_SNAKE_CASE			)






					def   __A			( self			):
						A__							:    Union[str, Any]        =		os.O_RDWR | os.O_CREAT | os.O_TRUNC
						A__							:    Dict        =		os.open(self._lock_file    ,       __SCREAMING_SNAKE_CASE			)

						try:
							fcntl.flock(__SCREAMING_SNAKE_CASE    ,       fcntl.LOCK_EX | fcntl.LOCK_NB			)
						except OSError:
							os.close(__SCREAMING_SNAKE_CASE			)
						else:
							A__							:    int        =		fd
						return None






					def   __A			( self			):
						# Do not remove the lockfile:
						#
						#   https://github.com/benediktschmitt/py-filelock/issues/31
						#   https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
						A__							:    Dict        =		self._lock_file_fd
						A__							:    Any        =		None
						fcntl.flock(__SCREAMING_SNAKE_CASE    ,       fcntl.LOCK_UN			)
						os.close(__SCREAMING_SNAKE_CASE			)
						return None



class     _a						(__snake_case				):


					'''simple docstring'''






					def   __A			( self			):
						A__							:    Any        =		os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
						try:
							A__							:    Dict        =		os.open(self._lock_file    ,       __SCREAMING_SNAKE_CASE			)
						except OSError:
							pass
						else:
							A__							:    List[Any]        =		fd
						return None






					def   __A			( self			):
						os.close(self._lock_file_fd			)
						A__							:    List[str]        =		None

						try:
							os.remove(self._lock_file			)
						# The file is already deleted and that's what we want.
						except OSError:
							pass
						return None



A_       : int         =			None

if msvcrt:
	A_       : Union[str, Any]         =			WindowsFileLock
elif fcntl:
	A_       : Union[str, Any]         =			UnixFileLock
else:
	A_       : Optional[int]         =			SoftFileLock

	if warnings is not None:
		warnings.warn('only soft file lock is available')











"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class      a						(					__snake_case  ,      unittest.TestCase		):
    SCREAMING_SNAKE_CASE						:     List[str]						= MobileBertTokenizer
    SCREAMING_SNAKE_CASE						:     int						= MobileBertTokenizerFast
    SCREAMING_SNAKE_CASE						:     Union[str, Any]						= True
    SCREAMING_SNAKE_CASE						:     Optional[Any]						= True
    SCREAMING_SNAKE_CASE						:     Dict						= filter_non_english
    SCREAMING_SNAKE_CASE						:     str						= """google/mobilebert-uncased"""






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Dict:
      super().setUp()

      lowerCamelCase_         =						[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      lowerCamelCase_         =						os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES['vocab_file']					)
      with open(self.vocab_file       ,						'w'       ,						encoding='utf-8'					) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)

      lowerCamelCase_         =						[
          (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
          for tokenizer_def in self.tokenizers_list
      ]






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]					)     ->		int:
      lowerCamelCase_         =						'UNwant\u00E9d,running'
      lowerCamelCase_         =						'unwanted, running'
      return input_text, output_text






    def    UpperCamelCase       (       self     :							Dict					)     ->		Any:
      lowerCamelCase_         =						self.tokenizer_class(self.vocab_file					)

      lowerCamelCase_         =						tokenizer.tokenize('UNwant\u00E9d,running'					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						['un', '##want', '##ed', ',', 'runn', '##ing']					)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)       ,						[9, 6, 7, 12, 10, 11]					)






    def    UpperCamelCase       (       self     :							Dict					)     ->		Dict:
      if not self.test_rust_tokenizer:
        return

      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      # With lower casing
      lowerCamelCase_         =						self.get_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						self.get_rust_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)       ,						['ah', '\u535A', '\u63A8', 'zz']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Optional[int]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['hello', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hällo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['h\u00E9llo']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		List[str]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Tuple:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		List[Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						never_split=['[UNK]']					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Any:
      lowerCamelCase_         =						['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      lowerCamelCase_         =						{}
      for i, token in enumerate(__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						i
      lowerCamelCase_         =						WordpieceTokenizer(vocab=__SCREAMING_SNAKE_CASE       ,						unk_token='[UNK]'					)

      self.assertListEqual(tokenizer.tokenize(''					)       ,						[]					)

      self.assertListEqual(tokenizer.tokenize('unwanted running'					)       ,						['un', '##want', '##ed', 'runn', '##ing']					)

      self.assertListEqual(tokenizer.tokenize('unwantedX running'					)       ,						['[UNK]', 'runn', '##ing']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Any:
      self.assertTrue(_is_whitespace(' '					)					)
      self.assertTrue(_is_whitespace('\t'					)					)
      self.assertTrue(_is_whitespace('\r'					)					)
      self.assertTrue(_is_whitespace('\n'					)					)
      self.assertTrue(_is_whitespace('\u00A0'					)					)

      self.assertFalse(_is_whitespace('A'					)					)
      self.assertFalse(_is_whitespace('-'					)					)






    def    UpperCamelCase       (       self     :							Union[str, Any]					)     ->		int:
      self.assertTrue(_is_control('\u0005'					)					)

      self.assertFalse(_is_control('A'					)					)
      self.assertFalse(_is_control(' '					)					)
      self.assertFalse(_is_control('\t'					)					)
      self.assertFalse(_is_control('\r'					)					)






    def    UpperCamelCase       (       self     :							str					)     ->		Optional[Any]:
      self.assertTrue(_is_punctuation('-'					)					)
      self.assertTrue(_is_punctuation('$'					)					)
      self.assertTrue(_is_punctuation('`'					)					)
      self.assertTrue(_is_punctuation('.'					)					)

      self.assertFalse(_is_punctuation('A'					)					)
      self.assertFalse(_is_punctuation(' '					)					)






    def    UpperCamelCase       (       self     :							int					)     ->		List[Any]:
      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)

      self.assertListEqual(
          [rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)






    @slow
    def    UpperCamelCase       (       self     :							Dict					)     ->		List[str]:
      lowerCamelCase_         =						self.tokenizer_class.from_pretrained('google/mobilebert-uncased'					)

      lowerCamelCase_         =						tokenizer.encode('sequence builders'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.encode('multi-sequence build'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
          lowerCamelCase_         =						tokenizer_r.encode_plus(
              __SCREAMING_SNAKE_CASE       ,						return_attention_mask=__SCREAMING_SNAKE_CASE       ,						return_token_type_ids=__SCREAMING_SNAKE_CASE       ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE       ,						)

          lowerCamelCase_         =						tokenizer_r.do_lower_case if hasattr(__SCREAMING_SNAKE_CASE       ,						'do_lower_case'					) else False
          lowerCamelCase_         =						(
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'A'),
                  ((1, 2), ','),
                  ((3, 5), 'na'),
                  ((5, 6), '##ï'),
                  ((6, 8), '##ve'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'Allen'),
                  ((21, 23), '##NL'),
                  ((23, 24), '##P'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'a'),
                  ((1, 2), ','),
                  ((3, 8), 'naive'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'allen'),
                  ((21, 23), '##nl'),
                  ((23, 24), '##p'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results]       ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
          self.assertEqual([e[0] for e in expected_results]       ,						tokens['offset_mapping']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		List[Any]:
      lowerCamelCase_         =						['的', '人', '有']
      lowerCamelCase_         =						''.join(__SCREAMING_SNAKE_CASE					)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						True
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that each Chinese character is not preceded by "##"
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						False
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that only the first Chinese character is not preceded by "##".
          lowerCamelCase_         =						[
              F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__SCREAMING_SNAKE_CASE					)
          ]
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)




