

import inspect
import unittest

from transformers import ConvNextVaConfig
from transformers.models.auto import get_values
from transformers.models.auto.modeling_auto import MODEL_FOR_BACKBONE_MAPPING_NAMES, MODEL_MAPPING_NAMES
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import ConvNextVaBackbone, ConvNextVaForImageClassification, ConvNextVaModel
  from transformers.models.convnextva.modeling_convnextva import CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
  from PIL import Image

  from transformers import AutoImageProcessor






class 							A_ :


     def __init__(							self			:		int			,  UpperCAmelCase			:		List[str]			,  UpperCAmelCase			:		Dict=1_3			,  UpperCAmelCase			:		Tuple=3_2			,  UpperCAmelCase			:		List[Any]=3			,  UpperCAmelCase			:		Any=4			,  UpperCAmelCase			:		str=[1_0, 2_0, 3_0, 4_0]			,  UpperCAmelCase			:		Dict=[2, 2, 3, 2]			,  UpperCAmelCase			:		Tuple=True			,  UpperCAmelCase			:		List[str]=True			,  UpperCAmelCase			:		Union[str, Any]=3_7			,  UpperCAmelCase			:		str="gelu"			,  UpperCAmelCase			:		Union[str, Any]=1_0			,  UpperCAmelCase			:		Optional[Any]=0.02			,  UpperCAmelCase			:		int=["stage2", "stage3", "stage4"]			,  UpperCAmelCase			:		Union[str, Any]=[2, 3, 4]			,  UpperCAmelCase			:		List[str]=None			,  )							->	Optional[Any]:
      __lowerCAmelCase:   List[str]   	=      parent
      __lowerCAmelCase:   int   	=      batch_size
      __lowerCAmelCase:   Optional[Any]   	=      image_size
      __lowerCAmelCase:   int   	=      num_channels
      __lowerCAmelCase:   Dict   	=      num_stages
      __lowerCAmelCase:   str   	=      hidden_sizes
      __lowerCAmelCase:   List[Any]   	=      depths
      __lowerCAmelCase:   Union[str, Any]   	=      is_training
      __lowerCAmelCase:   List[Any]   	=      use_labels
      __lowerCAmelCase:   int   	=      intermediate_size
      __lowerCAmelCase:   Optional[Any]   	=      hidden_act
      __lowerCAmelCase:   str   	=      num_labels
      __lowerCAmelCase:   str   	=      initializer_range
      __lowerCAmelCase:   Union[str, Any]   	=      out_features
      __lowerCAmelCase:   Union[str, Any]   	=      out_indices
      __lowerCAmelCase:   str   	=      scope


     def   UpperCAmelCase  (							self			:		Union[str, Any]					)							->	Any:
      __lowerCAmelCase:   Dict   	=      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]					)

      __lowerCAmelCase:   List[Any]   	=      None
      if self.use_labels:
       __lowerCAmelCase:   Optional[int]   	=      ids_tensor([self.batch_size]			,  self.num_labels					)

      __lowerCAmelCase:   Union[str, Any]   	=      self.get_config()

      return config, pixel_values, labels


     def   UpperCAmelCase  (							self			:		Optional[int]					)							->	List[str]:
      return ConvNextVaConfig(
          num_channels=self.num_channels			,  hidden_sizes=self.hidden_sizes			,  depths=self.depths			,  num_stages=self.num_stages			,  hidden_act=self.hidden_act			,  is_decoder=UpperCamelCase__			,  initializer_range=self.initializer_range			,  out_features=self.out_features			,  out_indices=self.out_indices			,  num_labels=self.num_labels			,  )


     def   UpperCAmelCase  (							self			:		Dict			,  UpperCAmelCase			:		Tuple			,  UpperCAmelCase			:		Optional[int]			,  UpperCAmelCase			:		Tuple					)							->	List[str]:
      __lowerCAmelCase:   Union[str, Any]   	=      ConvNextVaModel(config=UpperCamelCase__					)
      model.to(UpperCamelCase__					)
      model.eval()
      __lowerCAmelCase:   List[Any]   	=      model(UpperCamelCase__					)
      # expected last hidden states: B, C, H // 32, W // 32
      self.parent.assertEqual(
          result.last_hidden_state.shape			,  (self.batch_size, self.hidden_sizes[-1], self.image_size // 3_2, self.image_size // 3_2)			,  )


     def   UpperCAmelCase  (							self			:		str			,  UpperCAmelCase			:		Any			,  UpperCAmelCase			:		Optional[int]			,  UpperCAmelCase			:		Optional[Any]					)							->	Dict:
      __lowerCAmelCase:   Tuple   	=      ConvNextVaForImageClassification(UpperCamelCase__					)
      model.to(UpperCamelCase__					)
      model.eval()
      __lowerCAmelCase:   int   	=      model(UpperCamelCase__			,  labels=UpperCamelCase__					)
      self.parent.assertEqual(result.logits.shape			,  (self.batch_size, self.num_labels)					)


     def   UpperCAmelCase  (							self			:		int			,  UpperCAmelCase			:		List[str]			,  UpperCAmelCase			:		Optional[Any]			,  UpperCAmelCase			:		Union[str, Any]					)							->	Any:
      __lowerCAmelCase:   Optional[int]   	=      ConvNextVaBackbone(config=UpperCamelCase__					)
      model.to(UpperCamelCase__					)
      model.eval()
      __lowerCAmelCase:   int   	=      model(UpperCamelCase__					)

      # verify hidden states
      self.parent.assertEqual(len(result.feature_maps					)			,  len(config.out_features					)					)
      self.parent.assertListEqual(list(result.feature_maps[0].shape					)			,  [self.batch_size, self.hidden_sizes[1], 4, 4]					)

      # verify channels
      self.parent.assertEqual(len(model.channels					)			,  len(config.out_features					)					)
      self.parent.assertListEqual(model.channels			,  config.hidden_sizes[1:]					)

      # verify backbone works with out_features=None
      __lowerCAmelCase:   List[Any]   	=      None
      __lowerCAmelCase:   Any   	=      ConvNextVaBackbone(config=UpperCamelCase__					)
      model.to(UpperCamelCase__					)
      model.eval()
      __lowerCAmelCase:   Tuple   	=      model(UpperCamelCase__					)

      # verify feature maps
      self.parent.assertEqual(len(result.feature_maps					)			,  1					)
      self.parent.assertListEqual(list(result.feature_maps[0].shape					)			,  [self.batch_size, self.hidden_sizes[-1], 1, 1]					)

      # verify channels
      self.parent.assertEqual(len(model.channels					)			,  1					)
      self.parent.assertListEqual(model.channels			,  [config.hidden_sizes[-1]]					)


     def   UpperCAmelCase  (							self			:		int					)							->	Dict:
      __lowerCAmelCase:   List[Any]   	=      self.prepare_config_and_inputs()
      __lowerCAmelCase:   int   	=      config_and_inputs
      __lowerCAmelCase:   Optional[int]   	=      {"pixel_values": pixel_values}
      return config, inputs_dict


     def   UpperCAmelCase  (							self			:		Optional[int]					)							->	Union[str, Any]:
      __lowerCAmelCase:   str   	=      self.prepare_config_and_inputs()
      __lowerCAmelCase:   List[Any]   	=      config_and_inputs
      __lowerCAmelCase:   List[str]   	=      {"pixel_values": pixel_values, "labels": labels}
      return config, inputs_dict







@require_torch
class 							A_ (		__snake_case							,	__snake_case							,	unittest.TestCase					):
     _lowercase			:     str								=	(
         (
             ConvNextVaModel,
             ConvNextVaForImageClassification,
             ConvNextVaBackbone,
         )
         if is_torch_available()
         else ()
     )
     _lowercase			:     Optional[Any]								=	(
         {"""feature-extraction""": ConvNextVaModel, """image-classification""": ConvNextVaForImageClassification}
         if is_torch_available()
         else {}
     )

     _lowercase			:     Union[str, Any]								=	False
     _lowercase			:     Union[str, Any]								=	False
     _lowercase			:     int								=	False
     _lowercase			:     Union[str, Any]								=	False
     _lowercase			:     Dict								=	False


     def   UpperCAmelCase  (							self			:		Optional[Any]					)							->	Any:
      __lowerCAmelCase:   List[str]   	=      ConvNextVaModelTester(self					)
      __lowerCAmelCase:   Optional[Any]   	=      ConfigTester(self			,  config_class=UpperCamelCase__			,  has_text_modality=UpperCamelCase__			,  hidden_size=3_7					)


     def   UpperCAmelCase  (							self			:		Dict					)							->	str:
      self.create_and_test_config_common_properties()
      self.config_tester.create_and_test_config_to_json_string()
      self.config_tester.create_and_test_config_to_json_file()
      self.config_tester.create_and_test_config_from_and_save_pretrained()
      self.config_tester.create_and_test_config_with_num_labels()
      self.config_tester.check_config_can_be_init_without_params()
      self.config_tester.check_config_arguments_init()


     def   UpperCAmelCase  (							self			:		Tuple					)							->	Dict:
      return


     @unittest.skip(reason='ConvNextV2 does not use inputs_embeds'					)
     def   UpperCAmelCase  (							self			:		Optional[int]					)							->	List[Any]:
      pass


     @unittest.skip(reason='ConvNextV2 does not support input and output embeddings'					)
     def   UpperCAmelCase  (							self			:		Optional[int]					)							->	int:
      pass


     @unittest.skip(reason='ConvNextV2 does not use feedforward chunking'					)
     def   UpperCAmelCase  (							self			:		str					)							->	Any:
      pass


     def   UpperCAmelCase  (							self			:		str					)							->	Any:


      if not self.model_tester.is_training:
       return

      for model_class in self.all_model_classes:
       __lowerCAmelCase:   Dict   	=      self.model_tester.prepare_config_and_inputs_with_labels()
       __lowerCAmelCase:   Optional[int]   	=      True

       if model_class.__name__ in [
           *get_values(UpperCamelCase__					),
           *get_values(UpperCamelCase__					),
       ]:
        continue

       __lowerCAmelCase:   Optional[Any]   	=      model_class(UpperCamelCase__					)
       model.to(UpperCamelCase__					)
       model.train()
       __lowerCAmelCase:   str   	=      self._prepare_for_class(UpperCamelCase__			,  UpperCamelCase__			,  return_labels=UpperCamelCase__					)
       __lowerCAmelCase:   List[Any]   	=      model(**UpperCamelCase__					).loss
       loss.backward()


     def   UpperCAmelCase  (							self			:		Dict					)							->	str:


      if not self.model_tester.is_training:
       return

      for model_class in self.all_model_classes:
       __lowerCAmelCase:   int   	=      self.model_tester.prepare_config_and_inputs_with_labels()
       __lowerCAmelCase:   str   	=      False
       __lowerCAmelCase:   List[str]   	=      True

       if (
           model_class.__name__
           in [*get_values(UpperCamelCase__					), *get_values(UpperCamelCase__					)]
           or not model_class.supports_gradient_checkpointing
       ):
        continue

       __lowerCAmelCase:   Any   	=      model_class(UpperCamelCase__					)
       model.to(UpperCamelCase__					)
       model.gradient_checkpointing_enable()
       model.train()
       __lowerCAmelCase:   int   	=      self._prepare_for_class(UpperCamelCase__			,  UpperCamelCase__			,  return_labels=UpperCamelCase__					)
       __lowerCAmelCase:   Optional[Any]   	=      model(**UpperCamelCase__					).loss
       loss.backward()


     def   UpperCAmelCase  (							self			:		Dict					)							->	Dict:
      __lowerCAmelCase:   Optional[int]   	=      self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       __lowerCAmelCase:   str   	=      model_class(UpperCamelCase__					)
       __lowerCAmelCase:   Optional[Any]   	=      inspect.signature(model.forward					)
       # signature.parameters is an OrderedDict => so arg_names order is deterministic
       __lowerCAmelCase:   Optional[int]   	=      [*signature.parameters.keys()]

       __lowerCAmelCase:   Optional[Any]   	=      ["pixel_values"]
       self.assertListEqual(arg_names[:1]			,  UpperCamelCase__					)


     def   UpperCAmelCase  (							self			:		Any					)							->	int:
      __lowerCAmelCase:   int   	=      self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*UpperCamelCase__					)


     def   UpperCAmelCase  (							self			:		Optional[int]					)							->	Union[str, Any]:


      def check_hidden_states_output(UpperCAmelCase			:		List[str]			,  UpperCAmelCase			:		List[Any]			,  UpperCAmelCase			:		Any					):
       __lowerCAmelCase:   Optional[int]   	=      model_class(UpperCamelCase__					)
       model.to(UpperCamelCase__					)
       model.eval()

       with torch.no_grad():
        __lowerCAmelCase:   int   	=      model(**self._prepare_for_class(UpperCamelCase__			,  UpperCamelCase__					)					)

       __lowerCAmelCase:   str   	=      outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

       __lowerCAmelCase:   Optional[int]   	=      self.model_tester.num_stages
       self.assertEqual(len(UpperCamelCase__					)			,  expected_num_stages + 1					)

       # ConvNextV2's feature maps are of shape (batch_size, num_channels, height, width)
       self.assertListEqual(
           list(hidden_states[0].shape[-2:]					)			,  [self.model_tester.image_size // 4, self.model_tester.image_size // 4]			,  )

      __lowerCAmelCase:   Union[str, Any]   	=      self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       __lowerCAmelCase:   Tuple   	=      True
       check_hidden_states_output(UpperCamelCase__			,  UpperCamelCase__			,  UpperCamelCase__					)

       # check that output_hidden_states also work using config
       del inputs_dict["output_hidden_states"]
       __lowerCAmelCase:   Dict   	=      True

       check_hidden_states_output(UpperCamelCase__			,  UpperCamelCase__			,  UpperCamelCase__					)


     def   UpperCAmelCase  (							self			:		int					)							->	Any:
      __lowerCAmelCase:   Optional[Any]   	=      self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_image_classification(*UpperCamelCase__					)


     @slow
     def   UpperCAmelCase  (							self			:		Tuple					)							->	int:


      for model_name in CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       __lowerCAmelCase:   List[str]   	=      ConvNextVaModel.from_pretrained(UpperCamelCase__					)
       self.assertIsNotNone(UpperCamelCase__					)







def _a (       )    ->    str:

 """simple docstring"""






 __lowerCAmelCase:   str   	=      Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'		)
 return image






@require_torch
@require_vision
class 							A_ (		unittest.TestCase					):


     @cached_property
     def   UpperCAmelCase  (							self			:		List[Any]					)							->	int:
      return AutoImageProcessor.from_pretrained('facebook/convnextv2-tiny-1k-224'					) if is_vision_available() else None


     @slow
     def   UpperCAmelCase  (							self			:		Optional[int]					)							->	Optional[int]:
      __lowerCAmelCase:   str   	=      ConvNextVaForImageClassification.from_pretrained('facebook/convnextv2-tiny-1k-224'					).to(UpperCamelCase__					)

      __lowerCAmelCase:   Optional[int]   	=      self.default_image_processor
      __lowerCAmelCase:   Any   	=      prepare_img()
      __lowerCAmelCase:   List[Any]   	=      preprocessor(images=UpperCamelCase__			,  return_tensors='pt'					).to(UpperCamelCase__					)

      # forward pass
      with torch.no_grad():
       __lowerCAmelCase:   Optional[Any]   	=      model(**UpperCamelCase__					)

      # verify the logits
      __lowerCAmelCase:   List[str]   	=      torch.Size((1, 1_0_0_0)					)
      self.assertEqual(outputs.logits.shape			,  UpperCamelCase__					)

      __lowerCAmelCase:   str   	=      torch.tensor([0.9996, 0.1966, -0.4386]					).to(UpperCamelCase__					)
      self.assertTrue(torch.allclose(outputs.logits[0, :3]			,  UpperCamelCase__			,  atol=1E-4					)					)








"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class     snake_case	(				__snake_case,							unittest.TestCase  ):
				SCREAMING_SNAKE_CASE_   :      List[Any]         =  RoCBertTokenizer
				SCREAMING_SNAKE_CASE_   :      Any         =  None
				SCREAMING_SNAKE_CASE_   :      Any         =  False
				SCREAMING_SNAKE_CASE_   :      Dict         =  True
				SCREAMING_SNAKE_CASE_   :      Optional[int]         =  filter_non_english

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						super().setUp()

						__lowerCAmelCase:       Optional[Any]          =			["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
						__lowerCAmelCase:       List[Any]          =			{}
						__lowerCAmelCase:       Dict          =			{}
						for i, value in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       List[Any]          =			i
								__lowerCAmelCase:       Union[str, Any]          =			i
						__lowerCAmelCase:       List[str]          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["vocab_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_shape_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_pronunciation_file"])
						with open(self.vocab_file     ,  "w"     ,  encoding="utf-8") as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
						with open(self.word_shape_file     ,  "w"     ,  encoding="utf-8") as word_shape_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)
						with open(self.word_pronunciation_file     ,  "w"     ,  encoding="utf-8") as word_pronunciation_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)

				def 							lowercase_     (	self							:				Any)->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.tokenize("你好[SEP]你是谁")
						self.assertListEqual(UpperCamelCase__     ,  ["你", "好", "[SEP]", "你", "是", "谁"])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])

				def 							lowercase_     (	self							:				Optional[Any])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")     ,  ["ah", "\u535A", "\u63A8", "zz"])

				def 							lowercase_     (	self							:				str)->   Dict:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["hello", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Optional[int])->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hällo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["h\u00E9llo"])

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				str)->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Any)->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[int])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[int]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HäLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[Any])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HaLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Tuple)->   str:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  never_split=["[UNK]"])

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])

				def 							lowercase_     (	self							:				List[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

						__lowerCAmelCase:       int          =			{}
						for i, token in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       Optional[Any]          =			i
						__lowerCAmelCase:       str          =			RoCBertWordpieceTokenizer(vocab=UpperCamelCase__     ,  unk_token="[UNK]")

						self.assertListEqual(tokenizer.tokenize("")     ,  [])

						self.assertListEqual(tokenizer.tokenize("unwanted running")     ,  ["un", "##want", "##ed", "runn", "##ing"])

						self.assertListEqual(tokenizer.tokenize("unwantedX running")     ,  ["[UNK]", "runn", "##ing"])

				def 							lowercase_     (	self							:				Optional[Any])->   Dict:



						'''simple docstring'''


						self.assertTrue(_is_whitespace(" "))
						self.assertTrue(_is_whitespace("\t"))
						self.assertTrue(_is_whitespace("\r"))
						self.assertTrue(_is_whitespace("\n"))
						self.assertTrue(_is_whitespace("\u00A0"))

						self.assertFalse(_is_whitespace("A"))
						self.assertFalse(_is_whitespace("-"))

				def 							lowercase_     (	self							:				Dict)->   Optional[int]:



						'''simple docstring'''


						self.assertTrue(_is_control("\u0005"))

						self.assertFalse(_is_control("A"))
						self.assertFalse(_is_control(" "))
						self.assertFalse(_is_control("\t"))
						self.assertFalse(_is_control("\r"))

				def 							lowercase_     (	self							:				Union[str, Any])->   str:



						'''simple docstring'''


						self.assertTrue(_is_punctuation("-"))
						self.assertTrue(_is_punctuation("$"))
						self.assertTrue(_is_punctuation("`"))
						self.assertTrue(_is_punctuation("."))

						self.assertFalse(_is_punctuation("A"))
						self.assertFalse(_is_punctuation(" "))

				def 							lowercase_     (	self							:				Dict)->   int:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

						if self.test_rust_tokenizer:
								__lowerCAmelCase:       Any          =			self.get_rust_tokenizer()
								self.assertListEqual(
								    [rust_tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

				def 							lowercase_     (	self							:				Dict)->   Any:



						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       str          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
										__lowerCAmelCase:       Tuple          =			tokenizer_r.encode_plus(
										    UpperCamelCase__     ,  return_attention_mask=UpperCamelCase__     ,  return_token_type_ids=UpperCamelCase__     ,  return_offsets_mapping=UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__     ,  )

										__lowerCAmelCase:       str          =			tokenizer_r.do_lower_case if hasattr(UpperCamelCase__     ,  "do_lower_case") else False
										__lowerCAmelCase:       List[Any]          =			(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "Allen"),
										        ((2_1, 2_3), "##NL"),
										        ((2_3, 2_4), "##P"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "allen"),
										        ((2_1, 2_3), "##nl"),
										        ((2_3, 2_4), "##p"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]     ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
										self.assertEqual([e[0] for e in expected_results]     ,  tokens["offset_mapping"])

				def 							lowercase_     (	self							:				Union[str, Any])->   Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			["的", "人", "有"]
						__lowerCAmelCase:       int          =			"".join(UpperCamelCase__)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       Tuple          =			True
										__lowerCAmelCase:       str          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Union[str, Any]          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       List[Any]          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Any          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       List[str]          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

										__lowerCAmelCase:       int          =			False
										__lowerCAmelCase:       Any          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       str          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowerCAmelCase:       Dict          =			[
										    f"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCamelCase__)
										]
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

				@slow
				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       str          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Dict          =			tokenizer.encode("你好"     ,  add_special_tokens=UpperCamelCase__)
						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.encode("你是谁"     ,  add_special_tokens=UpperCamelCase__)

						__lowerCAmelCase:       Tuple          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__)
						__lowerCAmelCase:       List[Any]          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     ,  UpperCamelCase__)

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]




				def 							lowercase_     (	self							:				Tuple)->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			self.get_tokenizers(do_lower_case=UpperCamelCase__)
						for tokenizer in tokenizers:
								with self.subTest(f"{tokenizer.__class__.__name__}"):
										__lowerCAmelCase:       str          =			"你好，你是谁"
										__lowerCAmelCase:       Dict          =			tokenizer.tokenize(UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			tokenizer.convert_tokens_to_ids(UpperCamelCase__)
										__lowerCAmelCase:       Optional[Any]          =			tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			tokenizer.prepare_for_model(
										    UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			tokenizer.encode_plus(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										self.assertEqual(UpperCamelCase__     ,  UpperCamelCase__)


