
"""simple docstring"""




import importlib
import inspect
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
_A			:       Any   		=       """src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
_A			:       Any   		=       importlib.util.spec_from_file_location(
    """transformers""",
    os.path.join(PATH_TO_TRANSFORMERS, """__init__.py"""),
    submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
_A			:       List[str]   		=       spec.loader.load_module()

_A			:       Optional[Any]   		=       transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
_A			:       Dict   		=       re.compile("""\[(.+?)\]\((https://huggingface\.co/.+?)\)""")


_A			:       Dict   		=       {
    """CLIPConfigMixin""",
    """DecisionTransformerConfigMixin""",
    """EncoderDecoderConfigMixin""",
    """RagConfigMixin""",
    """SpeechEncoderDecoderConfigMixin""",
    """VisionEncoderDecoderConfigMixin""",
    """VisionTextDualEncoderConfigMixin""",
}


def   __magic_name__	(      ) ->				Optional[int]:
       lowercase    :				List[str]       		=							[]

       for config_class in list(CONFIG_MAPPING.values()  ):
              lowercase    :				Optional[int]       		=							False

              # source code of `config_class`
              lowercase    :				Optional[int]       		=							inspect.getsource(__snake_case  )
              lowercase    :				Any       		=							_re_checkpoint.findall(__snake_case  )

              for checkpoint in checkpoints:
                     # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
                     # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
                     lowercase		,    lowercase    :				List[str]       		=							checkpoint

                     # verify the checkpoint name corresponds to the checkpoint link
                     lowercase    :				Optional[Any]       		=							f"""https://huggingface.co/{ckpt_name}"""
                     if ckpt_link == ckpt_link_from_name:
                            lowercase    :				Any       		=							True
                            break

              lowercase    :				List[Any]       		=							config_class.__name__
              if not checkpoint_found and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
                     configs_without_checkpoint.append(__snake_case  )

       if len(__snake_case  ) > 0:
              lowercase    :				str       		=							"\n".join(sorted(__snake_case  )  )
              raise ValueError(f"""The following configurations don't contain any valid checkpoint:\n{message}"""  )


if __name__ == "__main__":
   check_config_docstrings_have_checkpoints()




"""simple docstring"""




import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin

@require_jieba
class 		a__						(      a_,			unittest.TestCase						):
   __lowerCAmelCase            =				CpmAntTokenizer
   __lowerCAmelCase            =				False
   def 			__magic_name__			(			self				):
          super().setUp()

          lowercase    :				List[str]       		=							[
              "<d>",
              "</d>",
              "<s>",
              "</s>",
              "</_>",
              "<unk>",
              "<pad>",
              "</n>",
              "我",
              "是",
              "C",
              "P",
              "M",
              "A",
              "n",
              "t",
          ]
          lowercase    :				Dict       		=							os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES["vocab_file"]				)
          with open(self.vocab_file	,    "w"	,    encoding="utf-8"				) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)

   @tooslow
   def 			__magic_name__			(			self				):
          lowercase    :				Tuple       		=							CpmAntTokenizer.from_pretrained("openbmb/cpm-ant-10b"				)
          lowercase    :				str       		=							"今天天气真好！"
          lowercase    :				Optional[int]       		=							["今天", "天气", "真", "好", "！"]
          lowercase    :				int       		=							tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				Any       		=							"今天天气真好！"
          lowercase    :				int       		=							[tokenizer.bos_token] + tokens

          lowercase    :				List[str]       		=							[6, 9_802, 14_962, 2_082, 831, 244]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_a				)	,    _a				)

          lowercase    :				str       		=							tokenizer.decode(_a				)
          self.assertEqual(_a	,    _a				)


