
"""simple docstring"""




from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import MagicMock, patch

from transformers import AutoModel, TFAutoModel
from transformers.onnx import FeaturesManager
from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch

@require_torch
@require_tf
class 		a__						(      a_						):
   def 			__magic_name__			(			self				):
          lowercase    :				Optional[Any]       		=							SMALL_MODEL_IDENTIFIER
          lowercase    :				Any       		=							"pt"
          lowercase    :				Optional[int]       		=							"tf"
   def 			__magic_name__			(			self	,    _a				):
          lowercase    :				Optional[int]       		=							AutoModel.from_pretrained(self.test_model				)
          model_pt.save_pretrained(_a				)
   def 			__magic_name__			(			self	,    _a				):
          lowercase    :				Union[str, Any]       		=							TFAutoModel.from_pretrained(self.test_model	,    from_pt=_a				)
          model_tf.save_pretrained(_a				)
   def 			__magic_name__			(			self				):
          lowercase    :				int       		=							"mock_framework"

          # Framework provided - return whatever the user provides
          lowercase    :				Optional[int]       		=							FeaturesManager.determine_framework(self.test_model	,    _a				)
          self.assertEqual(_a	,    _a				)

          # Local checkpoint and framework provided - return provided framework
          # PyTorch checkpoint
          with TemporaryDirectory() as local_pt_ckpt:
                 self._setup_pt_ckpt(_a				)
                 lowercase    :				List[Any]       		=							FeaturesManager.determine_framework(_a	,    _a				)
                 self.assertEqual(_a	,    _a				)

          # TensorFlow checkpoint
          with TemporaryDirectory() as local_tf_ckpt:
                 self._setup_tf_ckpt(_a				)
                 lowercase    :				str       		=							FeaturesManager.determine_framework(_a	,    _a				)
                 self.assertEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # PyTorch checkpoint
          with TemporaryDirectory() as local_pt_ckpt:
                 self._setup_pt_ckpt(_a				)
                 lowercase    :				Optional[Any]       		=							FeaturesManager.determine_framework(_a				)
                 self.assertEqual(_a	,    self.framework_pt				)

          # TensorFlow checkpoint
          with TemporaryDirectory() as local_tf_ckpt:
                 self._setup_tf_ckpt(_a				)
                 lowercase    :				List[Any]       		=							FeaturesManager.determine_framework(_a				)
                 self.assertEqual(_a	,    self.framework_tf				)

          # Invalid local checkpoint
          with TemporaryDirectory() as local_invalid_ckpt:
                 with self.assertRaises(_a				):
                        lowercase    :				Tuple       		=							FeaturesManager.determine_framework(_a				)

   def 			__magic_name__			(			self				):
          lowercase    :				Dict       		=							MagicMock(return_value=_a				)
          with patch("transformers.onnx.features.is_tf_available"	,    _a				):
                 lowercase    :				Optional[int]       		=							FeaturesManager.determine_framework(self.test_model				)
                 self.assertEqual(_a	,    self.framework_pt				)

          # PyTorch not in environment -> use TensorFlow
          lowercase    :				Optional[int]       		=							MagicMock(return_value=_a				)
          with patch("transformers.onnx.features.is_torch_available"	,    _a				):
                 lowercase    :				int       		=							FeaturesManager.determine_framework(self.test_model				)
                 self.assertEqual(_a	,    self.framework_tf				)

          # Both in environment -> use PyTorch
          lowercase    :				int       		=							MagicMock(return_value=_a				)
          lowercase    :				List[str]       		=							MagicMock(return_value=_a				)
          with patch("transformers.onnx.features.is_tf_available"	,    _a				), patch(
              "transformers.onnx.features.is_torch_available"	,    _a				):
                 lowercase    :				Any       		=							FeaturesManager.determine_framework(self.test_model				)
                 self.assertEqual(_a	,    self.framework_pt				)

          # Both not in environment -> raise error
          lowercase    :				List[str]       		=							MagicMock(return_value=_a				)
          lowercase    :				Dict       		=							MagicMock(return_value=_a				)
          with patch("transformers.onnx.features.is_tf_available"	,    _a				), patch(
              "transformers.onnx.features.is_torch_available"	,    _a				):
                 with self.assertRaises(_a				):
                        lowercase    :				Optional[Any]       		=							FeaturesManager.determine_framework(self.test_model				)




"""simple docstring"""




import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin

@require_jieba
class 		a__						(      a_,			unittest.TestCase						):
   __lowerCAmelCase            =				CpmAntTokenizer
   __lowerCAmelCase            =				False
   def 			__magic_name__			(			self				):
          super().setUp()

          lowercase    :				List[str]       		=							[
              "<d>",
              "</d>",
              "<s>",
              "</s>",
              "</_>",
              "<unk>",
              "<pad>",
              "</n>",
              "我",
              "是",
              "C",
              "P",
              "M",
              "A",
              "n",
              "t",
          ]
          lowercase    :				Dict       		=							os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES["vocab_file"]				)
          with open(self.vocab_file	,    "w"	,    encoding="utf-8"				) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)

   @tooslow
   def 			__magic_name__			(			self				):
          lowercase    :				Tuple       		=							CpmAntTokenizer.from_pretrained("openbmb/cpm-ant-10b"				)
          lowercase    :				str       		=							"今天天气真好！"
          lowercase    :				Optional[int]       		=							["今天", "天气", "真", "好", "！"]
          lowercase    :				int       		=							tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				Any       		=							"今天天气真好！"
          lowercase    :				int       		=							[tokenizer.bos_token] + tokens

          lowercase    :				List[str]       		=							[6, 9_802, 14_962, 2_082, 831, 244]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_a				)	,    _a				)

          lowercase    :				str       		=							tokenizer.decode(_a				)
          self.assertEqual(_a	,    _a				)


