

import argparse
import glob
import logging
import os
from argparse import Namespace
from importlib import import_module

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from seqeval.metrics import accuracy_score, fa_score, precision_score, recall_score
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader, TensorDataset
from utils_ner import TokenClassificationTask


_a		       =   logging.getLogger(__name__)






class 							A_ (		__snake_case					):
     _lowercase			:     Dict								=	"""token-classification"""


     def __init__(							self			:		str			,  UpperCAmelCase			:		Dict					)							->	List[Any]:


      if type(UpperCamelCase__					) == dict:
       __lowerCAmelCase:   Any   	=      Namespace(**UpperCamelCase__					)
      __lowerCAmelCase:   List[Any]   	=      import_module('tasks'					)
      try:
       __lowerCAmelCase:   int   	=      getattr(UpperCamelCase__			,  hparams.task_type					)
       __lowerCAmelCase:   TokenClassificationTask   	=      token_classification_task_clazz()
      except AttributeError:
       raise ValueError(
           F'''Task {hparams.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. '''
           F'''Available tasks classes are: {TokenClassificationTask.__subclasses__()}'''					)
      __lowerCAmelCase:   Dict   	=      self.token_classification_task.get_labels(hparams.labels					)
      __lowerCAmelCase:   Any   	=      CrossEntropyLoss().ignore_index
      super().__init__(UpperCamelCase__			,  len(self.labels					)			,  self.mode					)


     def   UpperCAmelCase  (							self			:		List[str]			,  **UpperCAmelCase			:		Optional[int]					)							->	Union[str, Any]:
      return self.model(**UpperCamelCase__					)


     def   UpperCAmelCase  (							self			:		Optional[int]			,  UpperCAmelCase			:		str			,  UpperCAmelCase			:		Optional[int]					)							->	List[str]:
      __lowerCAmelCase:   int   	=      {"input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3]}
      if self.config.model_type != "distilbert":
       __lowerCAmelCase:   int   	=      (
           batch[2] if self.config.model_type in ["bert", "xlnet"] else None
       )  # XLM and RoBERTa don"t use token_type_ids

      __lowerCAmelCase:   List[Any]   	=      self(**UpperCamelCase__					)
      __lowerCAmelCase:   List[str]   	=      outputs[0]
      # tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
      return {"loss": loss}


     def   UpperCAmelCase  (							self			:		Any					)							->	int:
      __lowerCAmelCase:   Any   	=      self.hparams
      for mode in ["train", "dev", "test"]:
       __lowerCAmelCase:   List[str]   	=      self._feature_file(UpperCamelCase__					)
       if os.path.exists(UpperCamelCase__					) and not args.overwrite_cache:
        logger.info('Loading features from cached file %s'			,  UpperCamelCase__					)
        __lowerCAmelCase:   Optional[Any]   	=      torch.load(UpperCamelCase__					)
       else:
        logger.info('Creating features from dataset file at %s'			,  args.data_dir					)
        __lowerCAmelCase:   List[str]   	=      self.token_classification_task.read_examples_from_file(args.data_dir			,  UpperCamelCase__					)
        __lowerCAmelCase:   Union[str, Any]   	=      self.token_classification_task.convert_examples_to_features(
            UpperCamelCase__			,  self.labels			,  args.max_seq_length			,  self.tokenizer			,  cls_token_at_end=bool(self.config.model_type in ['xlnet']					)			,  cls_token=self.tokenizer.cls_token			,  cls_token_segment_id=2 if self.config.model_type in ['xlnet'] else 0			,  sep_token=self.tokenizer.sep_token			,  sep_token_extra=UpperCamelCase__			,  pad_on_left=bool(self.config.model_type in ['xlnet']					)			,  pad_token=self.tokenizer.pad_token_id			,  pad_token_segment_id=self.tokenizer.pad_token_type_id			,  pad_token_label_id=self.pad_token_label_id			,  )
        logger.info('Saving features into cached file %s'			,  UpperCamelCase__					)
        torch.save(UpperCamelCase__			,  UpperCamelCase__					)


     def   UpperCAmelCase  (							self			:		List[str]			,  UpperCAmelCase			:		int			,  UpperCAmelCase			:		int			,  UpperCAmelCase			:		bool = False					)							->	DataLoader:
      __lowerCAmelCase:   Optional[Any]   	=      self._feature_file(UpperCamelCase__					)
      logger.info('Loading features from cached file %s'			,  UpperCamelCase__					)
      __lowerCAmelCase:   Optional[Any]   	=      torch.load(UpperCamelCase__					)
      __lowerCAmelCase:   List[Any]   	=      torch.tensor([f.input_ids for f in features]			,  dtype=torch.long					)
      __lowerCAmelCase:   Dict   	=      torch.tensor([f.attention_mask for f in features]			,  dtype=torch.long					)
      if features[0].token_type_ids is not None:
       __lowerCAmelCase:   str   	=      torch.tensor([f.token_type_ids for f in features]			,  dtype=torch.long					)
      else:
       __lowerCAmelCase:   Optional[int]   	=      torch.tensor([0 for f in features]			,  dtype=torch.long					)
       # HACK(we will not use this anymore soon)
      __lowerCAmelCase:   Optional[Any]   	=      torch.tensor([f.label_ids for f in features]			,  dtype=torch.long					)
      return DataLoader(
          TensorDataset(UpperCamelCase__			,  UpperCamelCase__			,  UpperCamelCase__			,  UpperCamelCase__					)			,  batch_size=UpperCamelCase__					)


     def   UpperCAmelCase  (							self			:		Any			,  UpperCAmelCase			:		Optional[int]			,  UpperCAmelCase			:		Tuple					)							->	Optional[int]:
      """Compute validation""" ""
      __lowerCAmelCase:   Union[str, Any]   	=      {"input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3]}
      if self.config.model_type != "distilbert":
       __lowerCAmelCase:   Optional[Any]   	=      (
           batch[2] if self.config.model_type in ["bert", "xlnet"] else None
       )  # XLM and RoBERTa don"t use token_type_ids
      __lowerCAmelCase:   Optional[int]   	=      self(**UpperCamelCase__					)
      __lowerCAmelCase:   Optional[Any]   	=      outputs[:2]
      __lowerCAmelCase:   List[Any]   	=      logits.detach().cpu().numpy()
      __lowerCAmelCase:   Dict   	=      inputs["labels"].detach().cpu().numpy()
      return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}


     def   UpperCAmelCase  (							self			:		Optional[Any]			,  UpperCAmelCase			:		Any					)							->	Union[str, Any]:
      __lowerCAmelCase:   List[Any]   	=      torch.stack([x['val_loss'] for x in outputs]					).mean()
      __lowerCAmelCase:   List[str]   	=      np.concatenate([x['pred'] for x in outputs]			,  axis=0					)
      __lowerCAmelCase:   str   	=      np.argmax(UpperCamelCase__			,  axis=2					)
      __lowerCAmelCase:   Union[str, Any]   	=      np.concatenate([x['target'] for x in outputs]			,  axis=0					)

      __lowerCAmelCase:   Optional[int]   	=      dict(enumerate(self.labels					)					)
      __lowerCAmelCase:   List[Any]   	=      [[] for _ in range(out_label_ids.shape[0]					)]
      __lowerCAmelCase:   List[Any]   	=      [[] for _ in range(out_label_ids.shape[0]					)]

      for i in range(out_label_ids.shape[0]					):
       for j in range(out_label_ids.shape[1]					):
        if out_label_ids[i, j] != self.pad_token_label_id:
         out_label_list[i].append(label_map[out_label_ids[i][j]]					)
         preds_list[i].append(label_map[preds[i][j]]					)

      __lowerCAmelCase:   int   	=      {
          "val_loss": val_loss_mean,
          "accuracy_score": accuracy_score(UpperCamelCase__			,  UpperCamelCase__					),
          "precision": precision_score(UpperCamelCase__			,  UpperCamelCase__					),
          "recall": recall_score(UpperCamelCase__			,  UpperCamelCase__					),
          "f1": fa_score(UpperCamelCase__			,  UpperCamelCase__					),
      }

      __lowerCAmelCase:   Optional[int]   	=      dict(results.items()					)
      __lowerCAmelCase:   Optional[Any]   	=      results
      return ret, preds_list, out_label_list


     def   UpperCAmelCase  (							self			:		str			,  UpperCAmelCase			:		Union[str, Any]					)							->	Optional[Any]:
      __lowerCAmelCase:   str   	=      self._eval_end(UpperCamelCase__					)
      __lowerCAmelCase:   Any   	=      ret["log"]
      return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}


     def   UpperCAmelCase  (							self			:		Tuple			,  UpperCAmelCase			:		Dict					)							->	Dict:
      __lowerCAmelCase:   Optional[int]   	=      self._eval_end(UpperCamelCase__					)

      # Converting to the dict required by pl
      # https://github.com/PyTorchLightning/pytorch-lightning/blob/master/\
      # pytorch_lightning/trainer/logging.py#L139
      __lowerCAmelCase:   Optional[Any]   	=      ret["log"]
      # `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
      return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}


     @staticmethod
     def   UpperCAmelCase  (							UpperCAmelCase			:		Union[str, Any]			,  UpperCAmelCase			:		str					)							->	Any:
      BaseTransformer.add_model_specific_args(UpperCamelCase__			,  UpperCamelCase__					)
      parser.add_argument(
          '--task_type'			,  default='NER'			,  type=UpperCamelCase__			,  help='Task type to fine tune in training (e.g. NER, POS, etc)'					)
      parser.add_argument(
          '--max_seq_length'			,  default=1_2_8			,  type=UpperCamelCase__			,  help=(
              'The maximum total input sequence length after tokenization. Sequences longer '
              'than this will be truncated, sequences shorter will be padded.'
          )			,  )

      parser.add_argument(
          '--labels'			,  default=''			,  type=UpperCamelCase__			,  help='Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.'			,  )
      parser.add_argument(
          '--gpus'			,  default=0			,  type=UpperCamelCase__			,  help='The number of GPUs allocated for this, it is by default 0 meaning none'			,  )

      parser.add_argument(
          '--overwrite_cache'			,  action='store_true'			,  help='Overwrite the cached training and evaluation sets'					)

      return parser


if __name__ == "__main__":
  _a		       =   argparse.ArgumentParser()
  add_generic_args(parser, os.getcwd())
  _a		       =   NERTransformer.add_model_specific_args(parser, os.getcwd())
  _a		       =   parser.parse_args()
  _a		       =   NERTransformer(args)
  _a		       =   generic_train(model, args)

  if args.do_predict:
    # See https://github.com/huggingface/transformers/issues/3159
    # pl use this default format to create a checkpoint:
    # https://github.com/PyTorchLightning/pytorch-lightning/blob/master\
    # /pytorch_lightning/callbacks/model_checkpoint.py#L322
    _a		       =   sorted(glob.glob(os.path.join(args.output_dir, '''checkpoint-epoch=*.ckpt'''), recursive=True))
    _a		       =   model.load_from_checkpoint(checkpoints[-1])
    trainer.test(model)








"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class     snake_case	(				__snake_case,							unittest.TestCase  ):
				SCREAMING_SNAKE_CASE_   :      List[Any]         =  RoCBertTokenizer
				SCREAMING_SNAKE_CASE_   :      Any         =  None
				SCREAMING_SNAKE_CASE_   :      Any         =  False
				SCREAMING_SNAKE_CASE_   :      Dict         =  True
				SCREAMING_SNAKE_CASE_   :      Optional[int]         =  filter_non_english

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						super().setUp()

						__lowerCAmelCase:       Optional[Any]          =			["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
						__lowerCAmelCase:       List[Any]          =			{}
						__lowerCAmelCase:       Dict          =			{}
						for i, value in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       List[Any]          =			i
								__lowerCAmelCase:       Union[str, Any]          =			i
						__lowerCAmelCase:       List[str]          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["vocab_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_shape_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_pronunciation_file"])
						with open(self.vocab_file     ,  "w"     ,  encoding="utf-8") as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
						with open(self.word_shape_file     ,  "w"     ,  encoding="utf-8") as word_shape_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)
						with open(self.word_pronunciation_file     ,  "w"     ,  encoding="utf-8") as word_pronunciation_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)

				def 							lowercase_     (	self							:				Any)->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.tokenize("你好[SEP]你是谁")
						self.assertListEqual(UpperCamelCase__     ,  ["你", "好", "[SEP]", "你", "是", "谁"])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])

				def 							lowercase_     (	self							:				Optional[Any])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")     ,  ["ah", "\u535A", "\u63A8", "zz"])

				def 							lowercase_     (	self							:				str)->   Dict:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["hello", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Optional[int])->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hällo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["h\u00E9llo"])

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				str)->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Any)->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[int])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[int]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HäLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[Any])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HaLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Tuple)->   str:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  never_split=["[UNK]"])

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])

				def 							lowercase_     (	self							:				List[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

						__lowerCAmelCase:       int          =			{}
						for i, token in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       Optional[Any]          =			i
						__lowerCAmelCase:       str          =			RoCBertWordpieceTokenizer(vocab=UpperCamelCase__     ,  unk_token="[UNK]")

						self.assertListEqual(tokenizer.tokenize("")     ,  [])

						self.assertListEqual(tokenizer.tokenize("unwanted running")     ,  ["un", "##want", "##ed", "runn", "##ing"])

						self.assertListEqual(tokenizer.tokenize("unwantedX running")     ,  ["[UNK]", "runn", "##ing"])

				def 							lowercase_     (	self							:				Optional[Any])->   Dict:



						'''simple docstring'''


						self.assertTrue(_is_whitespace(" "))
						self.assertTrue(_is_whitespace("\t"))
						self.assertTrue(_is_whitespace("\r"))
						self.assertTrue(_is_whitespace("\n"))
						self.assertTrue(_is_whitespace("\u00A0"))

						self.assertFalse(_is_whitespace("A"))
						self.assertFalse(_is_whitespace("-"))

				def 							lowercase_     (	self							:				Dict)->   Optional[int]:



						'''simple docstring'''


						self.assertTrue(_is_control("\u0005"))

						self.assertFalse(_is_control("A"))
						self.assertFalse(_is_control(" "))
						self.assertFalse(_is_control("\t"))
						self.assertFalse(_is_control("\r"))

				def 							lowercase_     (	self							:				Union[str, Any])->   str:



						'''simple docstring'''


						self.assertTrue(_is_punctuation("-"))
						self.assertTrue(_is_punctuation("$"))
						self.assertTrue(_is_punctuation("`"))
						self.assertTrue(_is_punctuation("."))

						self.assertFalse(_is_punctuation("A"))
						self.assertFalse(_is_punctuation(" "))

				def 							lowercase_     (	self							:				Dict)->   int:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

						if self.test_rust_tokenizer:
								__lowerCAmelCase:       Any          =			self.get_rust_tokenizer()
								self.assertListEqual(
								    [rust_tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

				def 							lowercase_     (	self							:				Dict)->   Any:



						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       str          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
										__lowerCAmelCase:       Tuple          =			tokenizer_r.encode_plus(
										    UpperCamelCase__     ,  return_attention_mask=UpperCamelCase__     ,  return_token_type_ids=UpperCamelCase__     ,  return_offsets_mapping=UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__     ,  )

										__lowerCAmelCase:       str          =			tokenizer_r.do_lower_case if hasattr(UpperCamelCase__     ,  "do_lower_case") else False
										__lowerCAmelCase:       List[Any]          =			(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "Allen"),
										        ((2_1, 2_3), "##NL"),
										        ((2_3, 2_4), "##P"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "allen"),
										        ((2_1, 2_3), "##nl"),
										        ((2_3, 2_4), "##p"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]     ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
										self.assertEqual([e[0] for e in expected_results]     ,  tokens["offset_mapping"])

				def 							lowercase_     (	self							:				Union[str, Any])->   Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			["的", "人", "有"]
						__lowerCAmelCase:       int          =			"".join(UpperCamelCase__)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       Tuple          =			True
										__lowerCAmelCase:       str          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Union[str, Any]          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       List[Any]          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Any          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       List[str]          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

										__lowerCAmelCase:       int          =			False
										__lowerCAmelCase:       Any          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       str          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowerCAmelCase:       Dict          =			[
										    f"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCamelCase__)
										]
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

				@slow
				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       str          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Dict          =			tokenizer.encode("你好"     ,  add_special_tokens=UpperCamelCase__)
						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.encode("你是谁"     ,  add_special_tokens=UpperCamelCase__)

						__lowerCAmelCase:       Tuple          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__)
						__lowerCAmelCase:       List[Any]          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     ,  UpperCamelCase__)

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]




				def 							lowercase_     (	self							:				Tuple)->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			self.get_tokenizers(do_lower_case=UpperCamelCase__)
						for tokenizer in tokenizers:
								with self.subTest(f"{tokenizer.__class__.__name__}"):
										__lowerCAmelCase:       str          =			"你好，你是谁"
										__lowerCAmelCase:       Dict          =			tokenizer.tokenize(UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			tokenizer.convert_tokens_to_ids(UpperCamelCase__)
										__lowerCAmelCase:       Optional[Any]          =			tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			tokenizer.prepare_for_model(
										    UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			tokenizer.encode_plus(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										self.assertEqual(UpperCamelCase__     ,  UpperCamelCase__)


