





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  __UpperCamelCase						(       A__  ,   unittest.TestCase						):
     __A  :     Optional[Any]            =      PreTrainedTokenizerFast
     __A  :     Optional[Any]            =      False
     __A  :     str            =      True
     __A  :     Optional[int]            =      """tokenizer_file"""





     def 		UpperCamelCase(    self				):
        _UpperCAmelCase											=      False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        _UpperCAmelCase											=      True

        _UpperCAmelCase											=      ['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
        _UpperCAmelCase											=      '''SaulLu/dummy-tokenizer-bytelevel-bpe'''

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        _UpperCAmelCase											=      [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        _UpperCAmelCase											=      PreTrainedTokenizerFast.from_pretrained(model_paths[0]				)
        tokenizer.save_pretrained(self.tmpdirname				)





     def 		UpperCamelCase(    self				):
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass





     def 		UpperCamelCase(    self				):
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass





     def 		UpperCamelCase(    self				):
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass





     def 		UpperCamelCase(    self				):
        # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
        pass





     def 		UpperCamelCase(    self				):
        _UpperCAmelCase											=      self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
              try:
                 _UpperCAmelCase											=      tempfile.mkdtemp()
                 _UpperCAmelCase											=      self.rust_tokenizer_class.from_pretrained(_UpperCamelCase							,       **_UpperCamelCase				)

                 tokenizer.save_pretrained(self.tmpdirname				)
                 super().test_training_new_tokenizer()
              finally:
                 # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                 # is restored
                 shutil.rmtree(self.tmpdirname				)
                 _UpperCAmelCase											=      tmpdirname_orig





     def 		UpperCamelCase(    self				):
        _UpperCAmelCase											=      self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
              try:
                 _UpperCAmelCase											=      tempfile.mkdtemp()
                 _UpperCAmelCase											=      self.rust_tokenizer_class.from_pretrained(_UpperCamelCase							,       **_UpperCamelCase				)

                 tokenizer.save_pretrained(self.tmpdirname				)
                 super().test_training_new_tokenizer_with_special_tokens_change()
              finally:
                 # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                 # is restored
                 shutil.rmtree(self.tmpdirname				)
                 _UpperCAmelCase											=      tmpdirname_orig





     def 		UpperCamelCase(    self				):
        _UpperCAmelCase											=      self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name				)

        _UpperCAmelCase											=      ('''a''' for _ in range(1000				))
        _UpperCAmelCase											=      tokenizer.train_new_from_iterator(text_iterator=_UpperCamelCase							,       length=1000							,       vocab_size=50				)

        _UpperCAmelCase											=      new_tokenizer.encode('''a🤗'''				)
        self.assertEqual(_UpperCamelCase							,       [64, 172, 253, 97, 245]				)




@require_tokenizers
class  __UpperCamelCase						(       unittest.TestCase						):





     def 		UpperCamelCase(    self				):
        _UpperCAmelCase											=      AutoTokenizer.from_pretrained('''bert-base-cased'''				)
        _UpperCAmelCase											=      json.loads(tokenizer._tokenizer.to_str()				)
        _UpperCAmelCase											=      len(_UpperCamelCase				)

        with tempfile.TemporaryDirectory() as tmp_dir:
           # Hack to save this in the tokenizer_config.json
           _UpperCAmelCase											=      ['''tokenizer.4.0.0.json''']
           tokenizer.save_pretrained(_UpperCamelCase				)
           json.dump(_UpperCamelCase							,       open(os.path.join(_UpperCamelCase							,       '''tokenizer.4.0.0.json'''				)							,       '''w'''				)				)

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           _UpperCAmelCase											=      AutoTokenizer.from_pretrained(_UpperCamelCase				)
           self.assertEqual(len(_UpperCamelCase				)							,       len(_UpperCamelCase				) + 1				)
           _UpperCAmelCase											=      json.loads(new_tokenizer._tokenizer.to_str()				)
           self.assertIn('''huggingface'''							,       json_tokenizer['''model''']['''vocab''']				)

           # Will need to be adjusted if we reach v42 and this test is still here.
           # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
           shutil.move(os.path.join(_UpperCamelCase							,       '''tokenizer.4.0.0.json'''				)							,       os.path.join(_UpperCamelCase							,       '''tokenizer.42.0.0.json'''				)				)
           _UpperCAmelCase											=      ['''tokenizer.42.0.0.json''']
           tokenizer.save_pretrained(_UpperCamelCase				)
           _UpperCAmelCase											=      AutoTokenizer.from_pretrained(_UpperCamelCase				)
           self.assertEqual(len(_UpperCamelCase				)							,       len(_UpperCamelCase				)				)
           _UpperCAmelCase											=      json.loads(new_tokenizer._tokenizer.to_str()				)
           self.assertNotIn('''huggingface'''							,       json_tokenizer['''model''']['''vocab''']				)





     def 		UpperCamelCase(    self				):
        # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
        _UpperCAmelCase											=      '''hf-internal-testing/test-two-tokenizers'''

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        _UpperCAmelCase											=      AutoTokenizer.from_pretrained(_UpperCamelCase				)
        self.assertEqual(len(_UpperCamelCase				)							,       28997				)
        _UpperCAmelCase											=      json.loads(tokenizer._tokenizer.to_str()				)
        self.assertIn('''huggingface'''							,       json_tokenizer['''model''']['''vocab''']				)

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        _UpperCAmelCase											=      '''3.0.0'''
        _UpperCAmelCase											=      old_transformers.models.auto.AutoTokenizer.from_pretrained(_UpperCamelCase				)
        self.assertEqual(len(_UpperCamelCase				)							,       28996				)
        _UpperCAmelCase											=      json.loads(old_tokenizer._tokenizer.to_str()				)
        self.assertNotIn('''huggingface'''							,       json_tokenizer['''model''']['''vocab''']				)




@require_tokenizers
class  __UpperCamelCase						(       unittest.TestCase						):





     def 		UpperCamelCase(    self				):
        # See https://github.com/huggingface/transformers/pull/12550
        # and https://github.com/huggingface/tokenizers/issues/537
        _UpperCAmelCase											=      PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''				)
        _UpperCAmelCase											=      '''The Matrix is a 1999 science fiction action film.'''

        with concurrent.futures.ThreadPoolExecutor() as executor:
           _UpperCAmelCase											=      [executor.submit(self.fetch							,       _UpperCamelCase							,       _UpperCamelCase				) for i in range(10				)]
           _UpperCAmelCase											=      [future.result() for future in futures]
           self.assertEqual(_UpperCamelCase							,       [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10				)]				)





     def 		UpperCamelCase(    self							,       _UpperCamelCase							,       _UpperCamelCase				):
        return tokenizer.encode(_UpperCamelCase							,       truncation='''longest_first'''							,       padding='''longest'''				)






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCAmelCase_											=  {
    "configuration_oneformer": ["ONEFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP", "OneFormerConfig"],
    "processing_oneformer": ["OneFormerProcessor"],
}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_											=  ["OneFormerImageProcessor"]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_											=  [
       "ONEFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
       "OneFormerForUniversalSegmentation",
       "OneFormerModel",
       "OneFormerPreTrainedModel",
   ]

if TYPE_CHECKING:
   from .configuration_oneformer import ONEFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, OneFormerConfig
   from .processing_oneformer import OneFormerProcessor

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .image_processing_oneformer import OneFormerImageProcessor
   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_oneformer import (
          ONEFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          OneFormerForUniversalSegmentation,
          OneFormerModel,
          OneFormerPreTrainedModel,
      )


else:
   import sys

   UpperCAmelCase_											=  _LazyModule(__name__, globals()["__file__"], _import_structure)