


'''simple docstring'''
import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin




@require_jieba
class 			lowerCamelCase_   (	__a     ,       unittest.TestCase      ):
      lowerCAmelCase__		     =     CpmAntTokenizer
      lowerCAmelCase__		     =     False






      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          super().setUp()

          UpperCAmelCase__			:		Union[str, Any]													=				[
              '''<d>''',
              '''</d>''',
              '''<s>''',
              '''</s>''',
              '''</_>''',
              '''<unk>''',
              '''<pad>''',
              '''</n>''',
              '''我''',
              '''是''',
              '''C''',
              '''P''',
              '''M''',
              '''A''',
              '''n''',
              '''t''',
          ]
          UpperCAmelCase__			:		Optional[Any]													=				os.path.join(self.tmpdirname						,						VOCAB_FILES_NAMES['''vocab_file'''] )
          with open(self.vocab_file						,						'''w'''						,						encoding='''utf-8''' ) as vocab_writer:
              vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )






      @tooslow
      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[Any]													=				CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b''' )
          UpperCAmelCase__			:		List[str]													=				'''今天天气真好！'''
          UpperCAmelCase__			:		List[str]													=				['''今天''', '''天气''', '''真''', '''好''', '''！''']
          UpperCAmelCase__			:		Tuple													=				tokenizer.tokenize(_A )
          self.assertListEqual(_A						,						_A )
          UpperCAmelCase__			:		Tuple													=				'''今天天气真好！'''
          UpperCAmelCase__			:		List[Any]													=				[tokenizer.bos_token] + tokens

          UpperCAmelCase__			:		Tuple													=				[6, 9_802, 14_962, 2_082, 831, 244]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_A )						,						_A )

          UpperCAmelCase__			:		List[str]													=				tokenizer.decode(_A )
          self.assertEqual(_A						,						_A )




'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__    					=      {
    '''configuration_distilbert''': [
        '''DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''DistilBertConfig''',
        '''DistilBertOnnxConfig''',
    ],
    '''tokenization_distilbert''': ['''DistilBertTokenizer'''],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase__    					=      ['''DistilBertTokenizerFast''']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase__    					=      [
      '''DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''DistilBertForMaskedLM''',
      '''DistilBertForMultipleChoice''',
      '''DistilBertForQuestionAnswering''',
      '''DistilBertForSequenceClassification''',
      '''DistilBertForTokenClassification''',
      '''DistilBertModel''',
      '''DistilBertPreTrainedModel''',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase__    					=      [
      '''TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''TFDistilBertForMaskedLM''',
      '''TFDistilBertForMultipleChoice''',
      '''TFDistilBertForQuestionAnswering''',
      '''TFDistilBertForSequenceClassification''',
      '''TFDistilBertForTokenClassification''',
      '''TFDistilBertMainLayer''',
      '''TFDistilBertModel''',
      '''TFDistilBertPreTrainedModel''',
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase__    					=      [
      '''FlaxDistilBertForMaskedLM''',
      '''FlaxDistilBertForMultipleChoice''',
      '''FlaxDistilBertForQuestionAnswering''',
      '''FlaxDistilBertForSequenceClassification''',
      '''FlaxDistilBertForTokenClassification''',
      '''FlaxDistilBertModel''',
      '''FlaxDistilBertPreTrainedModel''',
  ]


if TYPE_CHECKING:
  from .configuration_distilbert import (
      DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
      DistilBertConfig,
      DistilBertOnnxConfig,
  )
  from .tokenization_distilbert import DistilBertTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_distilbert_fast import DistilBertTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_distilbert import (
        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        DistilBertForMaskedLM,
        DistilBertForMultipleChoice,
        DistilBertForQuestionAnswering,
        DistilBertForSequenceClassification,
        DistilBertForTokenClassification,
        DistilBertModel,
        DistilBertPreTrainedModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_distilbert import (
        TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFDistilBertForMaskedLM,
        TFDistilBertForMultipleChoice,
        TFDistilBertForQuestionAnswering,
        TFDistilBertForSequenceClassification,
        TFDistilBertForTokenClassification,
        TFDistilBertMainLayer,
        TFDistilBertModel,
        TFDistilBertPreTrainedModel,
    )

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_distilbert import (
        FlaxDistilBertForMaskedLM,
        FlaxDistilBertForMultipleChoice,
        FlaxDistilBertForQuestionAnswering,
        FlaxDistilBertForSequenceClassification,
        FlaxDistilBertForTokenClassification,
        FlaxDistilBertModel,
        FlaxDistilBertPreTrainedModel,
    )

else:
  import sys

  UpperCamelCase__    					=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
