




import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase    : str		= get_tests_dir("fixtures/spiece.model")
@require_sentencepiece
@require_tokenizers
class       _UpperCamelCase   (					lowerCAmelCase						,							unittest.TestCase		):
      UpperCAmelCase_          =		ErnieMTokenizer
      UpperCAmelCase_          =		False
      UpperCAmelCase_          =		True
      UpperCAmelCase_          =		False
      UpperCAmelCase_          =		False
      def 						UpperCAmelCase_    (      self :Dict   )  ->      List[Any]:
       super().setUp()

       # We have a SentencePiece fixture for testing
       UpperCAmelCase__							=    ErnieMTokenizer(lowerCamelCase			,   unk_token="<unk>"			,   pad_token="<pad>"   )
       tokenizer.save_pretrained(self.tmpdirname   )
      def 						UpperCAmelCase_    (      self :int			,   lowerCamelCase :List[Any]   )  ->      Optional[int]:
       UpperCAmelCase__							=    "this is a test"
       UpperCAmelCase__							=    "this is a test"
       return input_text, output_text
      def 						UpperCAmelCase_    (      self :List[str]   )  ->      Optional[Any]:
       UpperCAmelCase__							=    "<pad>"
       UpperCAmelCase__							=    0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase   )			,   lowerCamelCase   )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase   )			,   lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :List[str]   )  ->      int:
       UpperCAmelCase__							=    list(self.get_tokenizer().get_vocab().keys()   )

       self.assertEqual(vocab_keys[0]			,   "<pad>"   )
       self.assertEqual(vocab_keys[1]			,   "<unk>"   )
       self.assertEqual(vocab_keys[-1]			,   "▁eloquent"   )
       self.assertEqual(len(lowerCamelCase   )			,   3_0000   )
      def 						UpperCAmelCase_    (      self :Any   )  ->      Union[str, Any]:
       self.assertEqual(self.get_tokenizer().vocab_size			,   3_0000   )
      def 						UpperCAmelCase_    (      self :Optional[int]   )  ->      List[str]:
       if not self.test_rust_tokenizer:
        return

       UpperCAmelCase__							=    self.get_tokenizer()
       UpperCAmelCase__							=    self.get_rust_tokenizer()

       UpperCAmelCase__							=    "I was born in 92000, and this is falsé."

       UpperCAmelCase__							=    tokenizer.tokenize(lowerCamelCase   )
       UpperCAmelCase__							=    rust_tokenizer.tokenize(lowerCamelCase   )
       self.assertListEqual(lowerCamelCase			,   lowerCamelCase   )

       UpperCAmelCase__							=    tokenizer.encode(lowerCamelCase			,   add_special_tokens=lowerCamelCase   )
       UpperCAmelCase__							=    rust_tokenizer.encode(lowerCamelCase			,   add_special_tokens=lowerCamelCase   )
       self.assertListEqual(lowerCamelCase			,   lowerCamelCase   )

       UpperCAmelCase__							=    self.get_rust_tokenizer()
       UpperCAmelCase__							=    tokenizer.encode(lowerCamelCase   )
       UpperCAmelCase__							=    rust_tokenizer.encode(lowerCamelCase   )
       self.assertListEqual(lowerCamelCase			,   lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :Dict   )  ->      Dict:
       UpperCAmelCase__							=    ErnieMTokenizer(lowerCamelCase			,   do_lower_case=lowerCamelCase			,   unk_token="<unk>"			,   pad_token="<pad>"   )

       UpperCAmelCase__							=    tokenizer.tokenize("This is a test"   )
       self.assertListEqual(lowerCamelCase			,   ["▁this", "▁is", "▁a", "▁test"]   )

       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase   )			,   [48, 25, 21, 1289]   )

       UpperCAmelCase__							=    tokenizer.tokenize("I was born in 92000, and this is falsé."   )
       # ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
       self.assertListEqual(
           lowerCamelCase			,   ["▁i", "▁was", "▁born", "▁in", "9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "."]   )
       UpperCAmelCase__							=    tokenizer.convert_tokens_to_ids(lowerCamelCase   )
       self.assertListEqual(lowerCamelCase			,   [31, 23, 386, 19, 518, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]   )

       UpperCAmelCase__							=    tokenizer.convert_ids_to_tokens(lowerCamelCase   )
       self.assertListEqual(
           lowerCamelCase			,   ["▁i", "▁was", "▁born", "▁in", "9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "."]			,   )
      def 						UpperCAmelCase_    (      self :Union[str, Any]   )  ->      Dict:
       UpperCAmelCase__							=    ErnieMTokenizer(lowerCamelCase			,   unk_token="<unk>"			,   pad_token="<pad>"   )

       UpperCAmelCase__							=    tokenizer.encode("sequence builders"   )
       UpperCAmelCase__							=    tokenizer.encode("multi-sequence build"   )

       UpperCAmelCase__							=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase			,   lowerCamelCase   )

       assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
       assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
           tokenizer.sep_token_id
       ] + text_a + [tokenizer.sep_token_id]







      @slow
      def 						UpperCAmelCase_    (      self :Dict   )  ->      Tuple:
       # fmt: off
       UpperCAmelCase__							=    {"input_ids": [[0, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 9, 304, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 5, 5, 5, 16, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 6460, 1328, 4589, 42, 12_2009, 11_5774, 23, 3559, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [0, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=lowerCamelCase			,   model_name="susnato/ernie-m-base_pytorch"			,   sequences=[
               "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "
               "general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural "
               "Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained "
               "models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.",
               "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "
               "conditioning on both left and right context in all layers.",
               "The quick brown fox jumps over the lazy dog.",
           ]			,   )











import argparse
import glob
import inspect
import os
import re
import unittest

from get_test_info import get_test_classes

from tests.test_pipeline_mixin import pipeline_test_mapping


_lowerCAmelCase    : int		= {}
for task, _ in pipeline_test_mapping.items():
  _lowerCAmelCase    : List[str]		= {"pt": None, "tf": None}


# DO **NOT** add item to this set (unless the reason is approved)
_lowerCAmelCase    : Optional[int]		= {
    "tests/models/esm/test_modeling_esmfold.py",  # The pipeline test mapping is added to `test_modeling_esm.py`
}



def     lowerCAmelCase		(		_lowerCAmelCase				: Union[str, Any]						):




 """simple docstring"""







 if "ModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
  return "pt"
 elif "TFModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
  return "tf"
 elif "FlaxModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
  return "flax"
 else:
  return None



def     lowerCAmelCase		(		_lowerCAmelCase				: str    ,				_lowerCAmelCase				: Optional[Any]						):




 """simple docstring"""







 if PIPELINE_TEST_MAPPING[task].get(_lowerCAmelCase    ,				_lowerCAmelCase						) is not None:
  return PIPELINE_TEST_MAPPING[task][framework]

 UpperCAmelCase__							=    pipeline_test_mapping[task]["test"]
 UpperCAmelCase__							=    None

 if framework == "pt":
  UpperCAmelCase__							=    getattr(_lowerCAmelCase    ,				"model_mapping"    ,				_lowerCAmelCase						)
 elif framework == "tf":
  UpperCAmelCase__							=    getattr(_lowerCAmelCase    ,				"tf_model_mapping"    ,				_lowerCAmelCase						)

 if mapping is not None:
  UpperCAmelCase__							=    dict(mapping.items()						)

 # cache the results
 UpperCAmelCase__							=    mapping
 return mapping



def     lowerCAmelCase		(		_lowerCAmelCase				: Tuple    ,				_lowerCAmelCase				: List[str]						):




 """simple docstring"""







 UpperCAmelCase__							=    get_framework(_lowerCAmelCase						)
 if framework is None:
  return None
 UpperCAmelCase__							=    get_mapping_for_task(_lowerCAmelCase    ,				_lowerCAmelCase						)
 if mapping is None:
  return None

 UpperCAmelCase__							=    list({model_class.config_class for model_class in test_class.all_model_classes}						)
 if len(_lowerCAmelCase						) != 1:
  raise ValueError("There should be exactly one configuration class from `test_class.all_model_classes`."						)

 # This could be a list/tuple of model classes, but it's rare.
 UpperCAmelCase__							=    mapping.get(config_classes[0]    ,				_lowerCAmelCase						)
 if isinstance(_lowerCAmelCase    ,				(tuple, list)						):
  UpperCAmelCase__							=    sorted(_lowerCAmelCase    ,				key=lambda _lowerCAmelCase						: x.__name__						)

 return model_class



def     lowerCAmelCase		(		_lowerCAmelCase				: str						):




 """simple docstring"""







 UpperCAmelCase__							=    [(task, get_model_for_pipeline_test(_lowerCAmelCase    ,				_lowerCAmelCase						)) for task in pipeline_test_mapping]
 UpperCAmelCase__							=    sorted([(task, model) for task, model in mapping if model is not None]    ,				key=lambda _lowerCAmelCase						: x[0]						)

 return dict(_lowerCAmelCase						)



def     lowerCAmelCase		(		_lowerCAmelCase				: Optional[Any]						):




 """simple docstring"""







 UpperCAmelCase__							=    get_framework(_lowerCAmelCase						)
 if framework == "pt":
  UpperCAmelCase__							=    "torch"
 UpperCAmelCase__							=    "{}"

 UpperCAmelCase__							=    get_pipeline_model_mapping(_lowerCAmelCase						)
 if len(_lowerCAmelCase						) == 0:
  return ""

 UpperCAmelCase__							=    []
 for task, model_classes in mapping.items():
  if isinstance(_lowerCAmelCase    ,				(tuple, list)						):
   # A list/tuple of model classes
   UpperCAmelCase__							=    "(" + ", ".join([x.__name__ for x in model_classes]						) + ")"
  else:
   # A single model class
   UpperCAmelCase__							=    model_classes.__name__
  texts.append(F'''"{task}": {value}'''						)
 UpperCAmelCase__							=    "{" + ", ".join(_lowerCAmelCase						) + "}"
 UpperCAmelCase__							=    F'''pipeline_model_mapping = {text} if is_{framework}_available() else {default_value}'''

 return text



def     lowerCAmelCase		(		_lowerCAmelCase				: Any						):




 """simple docstring"""







 UpperCAmelCase__							=    {"ModelTesterMixin", "TFModelTesterMixin", "FlaxModelTesterMixin"}
 if not issubclass(_lowerCAmelCase    ,				unittest.TestCase						):
  return False
 return len(base_class_names.intersection([x.__name__ for x in test_class.__bases__]						)						) > 0



def     lowerCAmelCase		(		_lowerCAmelCase				: Union[str, Any]						):




 """simple docstring"""







 UpperCAmelCase__							=    [x for x in get_test_classes(_lowerCAmelCase						) if is_valid_test_class(_lowerCAmelCase						)]

 UpperCAmelCase__							=    None
 for test_class in test_classes:
  # If a test class has defined `pipeline_model_mapping`, let's take it
  if getattr(_lowerCAmelCase    ,				"pipeline_model_mapping"    ,				_lowerCAmelCase						) is not None:
   UpperCAmelCase__							=    test_class
   break
    # Take the test class with the shortest name (just a heuristic)
 if target_test_class is None and len(_lowerCAmelCase						) > 0:
  UpperCAmelCase__							=    sorted(_lowerCAmelCase    ,				key=lambda _lowerCAmelCase						: (len(x.__name__						), x.__name__)						)[0]

 return target_test_class



def     lowerCAmelCase		(		_lowerCAmelCase				: int    ,				_lowerCAmelCase				: Optional[int]    ,				_lowerCAmelCase				: str						):




 """simple docstring"""







 UpperCAmelCase__							=    start_idx
 for idx, line in enumerate(lines[start_idx:]						):
  UpperCAmelCase__							=    len(_lowerCAmelCase						) - len(line.lstrip()						)
  if idx == 0 or indent > indent_level or (indent == indent_level and line.strip() == ")"):
   UpperCAmelCase__							=    start_idx + idx
  elif idx > 0 and indent <= indent_level:
   # Outside the definition block of `pipeline_model_mapping`
   break

 return end_idx



def     lowerCAmelCase		(		_lowerCAmelCase				: int    ,				_lowerCAmelCase				: str=False						):




 """simple docstring"""







 if getattr(_lowerCAmelCase    ,				"pipeline_model_mapping"    ,				_lowerCAmelCase						) is not None:
  if not overwrite:
   return "", -1

 UpperCAmelCase__							=    get_pipeline_model_mapping_string(_lowerCAmelCase						)
 if len(_lowerCAmelCase						) == 0:
  return "", -1
 UpperCAmelCase__							=    line_to_add + "\n"

 # The code defined the class `test_class`
 UpperCAmelCase__			,			UpperCAmelCase__							=    inspect.getsourcelines(_lowerCAmelCase						)
 # `inspect` gives the code for an object, including decorator(s) if any.
 # We (only) need the exact line of the class definition.
 for idx, line in enumerate(_lowerCAmelCase						):
  if line.lstrip().startswith("class "						):
   UpperCAmelCase__							=    class_lines[idx:]
   class_start_line_no += idx
   break
 UpperCAmelCase__							=    class_start_line_no + len(_lowerCAmelCase						) - 1

 # The index in `class_lines` that starts the definition of `all_model_classes`, `all_generative_model_classes` or
 # `pipeline_model_mapping`. This assumes they are defined in such order, and we take the start index of the last
 # block that appears in a `test_class`.
 UpperCAmelCase__							=    None
 # The indent level of the line at `class_lines[start_idx]` (if defined)
 UpperCAmelCase__							=    0
 # To record if `pipeline_model_mapping` is found in `test_class`.
 UpperCAmelCase__							=    None
 for idx, line in enumerate(_lowerCAmelCase						):
  if line.strip().startswith("all_model_classes = "						):
   UpperCAmelCase__							=    len(_lowerCAmelCase						) - len(line.lstrip()						)
   UpperCAmelCase__							=    idx
  elif line.strip().startswith("all_generative_model_classes = "						):
   UpperCAmelCase__							=    len(_lowerCAmelCase						) - len(line.lstrip()						)
   UpperCAmelCase__							=    idx
  elif line.strip().startswith("pipeline_model_mapping = "						):
   UpperCAmelCase__							=    len(_lowerCAmelCase						) - len(line.lstrip()						)
   UpperCAmelCase__							=    idx
   UpperCAmelCase__							=    line
   break

 if start_idx is None:
  return "", -1
 # Find the ending index (inclusive) of the above found block.
 UpperCAmelCase__							=    find_block_ending(_lowerCAmelCase    ,				_lowerCAmelCase    ,				_lowerCAmelCase						)

 # Extract `is_xxx_available()` from existing blocks: some models require specific libraries like `timm` and use
 # `is_timm_available()` instead of `is_torch_available()`.
 # Keep leading and trailing whitespaces
 UpperCAmelCase__							=    re.compile(R"\s(is_\S+?_available\(\))\s"						)
 for line in class_lines[start_idx : end_idx + 1]:
  UpperCAmelCase__							=    r.search(_lowerCAmelCase						)
  if backend_condition is not None:
   # replace the leading and trailing whitespaces to the space character " ".
   UpperCAmelCase__							=    " " + backend_condition[0][1:-1] + " "
   UpperCAmelCase__							=    r.sub(_lowerCAmelCase    ,				_lowerCAmelCase						)
   break

 if def_line is None:
  # `pipeline_model_mapping` is not defined. The target index is set to the ending index (inclusive) of
  # `all_model_classes` or `all_generative_model_classes`.
  UpperCAmelCase__							=    end_idx
 else:
  # `pipeline_model_mapping` is defined. The target index is set to be one **BEFORE** its start index.
  UpperCAmelCase__							=    start_idx - 1
  # mark the lines of the currently existing `pipeline_model_mapping` to be removed.
  for idx in range(_lowerCAmelCase    ,				end_idx + 1						):
   # These lines are going to be removed before writing to the test file.
   UpperCAmelCase__							=    None  # noqa

    # Make sure the test class is a subclass of `PipelineTesterMixin`.
 UpperCAmelCase__							=    [x.__name__ for x in test_class.__bases__]
 if "PipelineTesterMixin" not in parent_classes:
  # Put `PipelineTesterMixin` just before `unittest.TestCase`
  UpperCAmelCase__							=    [x for x in parent_classes if x != "TestCase"] + ["PipelineTesterMixin"]
  if "TestCase" in parent_classes:
   # Here we **assume** the original string is always with `unittest.TestCase`.
   _parent_classes.append("unittest.TestCase"						)
  UpperCAmelCase__							=    ", ".join(_parent_classes						)
  for idx, line in enumerate(_lowerCAmelCase						):
   # Find the ending of the declaration of `test_class`
   if line.strip().endswith("):"						):
    # mark the lines of the declaration of `test_class` to be removed
    for _idx in range(idx + 1						):
     UpperCAmelCase__							=    None  # noqa
    break
        # Add the new, one-line, class declaration for `test_class`
  UpperCAmelCase__							=    F'''class {test_class.__name__}({parent_classes}):\n'''

 # Add indentation
 UpperCAmelCase__							=    " " * indent_level + line_to_add
 # Insert `pipeline_model_mapping` to `class_lines`.
 # (The line at `target_idx` should be kept by definition!)
 UpperCAmelCase__							=    class_lines[: target_idx + 1] + [line_to_add] + class_lines[target_idx + 1 :]
 # Remove the lines that are marked to be removed
 UpperCAmelCase__							=    [x for x in class_lines if x is not None]

 # Move from test class to module (in order to write to the test file)
 UpperCAmelCase__							=    inspect.getsourcelines(inspect.getmodule(_lowerCAmelCase						)						)[0]
 # Be careful with the 1-off between line numbers and array indices
 UpperCAmelCase__							=    module_lines[: class_start_line_no - 1] + class_lines + module_lines[class_end_line_no:]
 UpperCAmelCase__							=    "".join(_lowerCAmelCase						)

 UpperCAmelCase__							=    inspect.getsourcefile(_lowerCAmelCase						)
 with open(_lowerCAmelCase    ,				"w"    ,				encoding="UTF-8"    ,				newline="\n"						) as fp:
  fp.write(_lowerCAmelCase						)

 return line_to_add



def     lowerCAmelCase		(		_lowerCAmelCase				: Tuple    ,				_lowerCAmelCase				: List[Any]=False						):




 """simple docstring"""







 UpperCAmelCase__							=    find_test_class(_lowerCAmelCase						)
 if test_class:
  add_pipeline_model_mapping(_lowerCAmelCase    ,				overwrite=_lowerCAmelCase						)


if __name__ == "__main__":
  _lowerCAmelCase    : Tuple		= argparse.ArgumentParser()
  parser.add_argument(
      "--test_file", type=str, help="A path to the test file, starting with the repository's `tests` directory."
  )
  parser.add_argument(
      "--all",
      action="store_true",
      help="If to check and modify all test files.",
  )
  parser.add_argument(
      "--overwrite",
      action="store_true",
      help="If to overwrite a test class if it has already defined `pipeline_model_mapping`.",
  )
  _lowerCAmelCase    : Optional[Any]		= parser.parse_args()

  if not args.all and not args.test_file:
    raise ValueError("Please specify either `test_file` or pass `--all` to check/modify all test files.")
  elif args.all and args.test_file:
    raise ValueError("Only one of `--test_file` and `--all` could be specified.")

  _lowerCAmelCase    : List[str]		= []
  if args.test_file:
    _lowerCAmelCase    : Any		= [args.test_file]
  else:
    _lowerCAmelCase    : Any		= os.path.join("tests", "models", "**", "test_modeling_*.py")
    for test_file in glob.glob(pattern):
      # `Flax` is not concerned at this moment
      if not test_file.startswith("test_modeling_flax_"):
        test_files.append(test_file)

  for test_file in test_files:
    if test_file in TEST_FILE_TO_IGNORE:
      print(F'''[SKIPPED] {test_file} is skipped as it is in `TEST_FILE_TO_IGNORE` in the file {__file__}.''')
      continue
    add_pipeline_model_mapping_to_test_file(test_file, overwrite=args.overwrite)





