


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class    A_   ( a_      ,						unittest.TestCase     ):







 """simple docstring"""





 _UpperCamelCase				: Tuple      =					PreTrainedTokenizerFast
 _UpperCamelCase				: Union[str, Any]      =					False
 _UpperCamelCase				: int      =					True
 _UpperCamelCase				: Optional[int]      =					'''tokenizer_file'''
 def   _UpperCamelCase    (			self					: str				)    ->			Dict:
      UpperCAmelCase						:				Optional[Any]		  =			False  # because we don't have pretrained_vocab_files_map
      super().setUp()
      UpperCAmelCase						:				List[Any]		  =			True

      UpperCAmelCase						:				Any		  =			["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
      UpperCAmelCase						:				Tuple		  =			"SaulLu/dummy-tokenizer-bytelevel-bpe"

      # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
      UpperCAmelCase						:				Optional[int]		  =			[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

      UpperCAmelCase						:				int		  =			PreTrainedTokenizerFast.from_pretrained(model_paths[0]				)
      tokenizer.save_pretrained(self.tmpdirname				)
 def   _UpperCamelCase    (			self					: Optional[Any]				)    ->			List[str]:
      # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
      # model
      pass
 def   _UpperCamelCase    (			self					: int				)    ->			int:
      # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
      # model
      pass
 def   _UpperCamelCase    (			self					: List[str]				)    ->			str:
      # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
      # model
      pass
 def   _UpperCamelCase    (			self					: Tuple				)    ->			List[str]:
      # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
      pass
 def   _UpperCamelCase    (			self					: str				)    ->			Tuple:
      UpperCAmelCase						:				Dict		  =			self.tmpdirname
      # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'				):
                try:
                     UpperCAmelCase						:				Optional[Any]		  =			tempfile.mkdtemp()
                     UpperCAmelCase						:				List[str]		  =			self.rust_tokenizer_class.from_pretrained(_a       ,    **_a				)

                     tokenizer.save_pretrained(self.tmpdirname				)
                     super().test_training_new_tokenizer()
                finally:
                     # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                     # is restored
                     shutil.rmtree(self.tmpdirname				)
                     UpperCAmelCase						:				Dict		  =			tmpdirname_orig
 def   _UpperCamelCase    (			self					: Optional[int]				)    ->			Optional[int]:
      UpperCAmelCase						:				int		  =			self.tmpdirname
      # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'				):
                try:
                     UpperCAmelCase						:				List[str]		  =			tempfile.mkdtemp()
                     UpperCAmelCase						:				Union[str, Any]		  =			self.rust_tokenizer_class.from_pretrained(_a       ,    **_a				)

                     tokenizer.save_pretrained(self.tmpdirname				)
                     super().test_training_new_tokenizer_with_special_tokens_change()
                finally:
                     # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                     # is restored
                     shutil.rmtree(self.tmpdirname				)
                     UpperCAmelCase						:				Optional[Any]		  =			tmpdirname_orig

 def   _UpperCamelCase    (			self					: Union[str, Any]				)    ->			int:
      UpperCAmelCase						:				Tuple		  =			self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name				)

      UpperCAmelCase						:				Dict		  =			("a" for _ in range(1_0_0_0				))
      UpperCAmelCase						:				Optional[int]		  =			tokenizer.train_new_from_iterator(text_iterator=_a       ,    length=1_0_0_0       ,    vocab_size=5_0				)

      UpperCAmelCase						:				List[str]		  =			new_tokenizer.encode("a🤗"				)
      self.assertEqual(_a       ,    [6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]				)




@require_tokenizers
class    A_   ( unittest.TestCase     ):







 """simple docstring"""





 def   _UpperCamelCase    (			self					: List[str]				)    ->			Union[str, Any]:
      UpperCAmelCase						:				Union[str, Any]		  =			AutoTokenizer.from_pretrained("bert-base-cased"				)
      UpperCAmelCase						:				Optional[Any]		  =			json.loads(tokenizer._tokenizer.to_str()				)
      UpperCAmelCase						:				Dict		  =			len(_a				)

      with tempfile.TemporaryDirectory() as tmp_dir:
           # Hack to save this in the tokenizer_config.json
           UpperCAmelCase						:				Any		  =			["tokenizer.4.0.0.json"]
           tokenizer.save_pretrained(_a				)
           json.dump(_a       ,    open(os.path.join(_a       ,    "tokenizer.4.0.0.json"				)       ,    "w"				)				)

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           UpperCAmelCase						:				List[str]		  =			AutoTokenizer.from_pretrained(_a				)
           self.assertEqual(len(_a				)       ,    len(_a				) + 1				)
           UpperCAmelCase						:				List[Any]		  =			json.loads(new_tokenizer._tokenizer.to_str()				)
           self.assertIn("huggingface"       ,    json_tokenizer["model"]["vocab"]				)

           # Will need to be adjusted if we reach v42 and this test is still here.
           # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
           shutil.move(os.path.join(_a       ,    "tokenizer.4.0.0.json"				)       ,    os.path.join(_a       ,    "tokenizer.42.0.0.json"				)				)
           UpperCAmelCase						:				Any		  =			["tokenizer.42.0.0.json"]
           tokenizer.save_pretrained(_a				)
           UpperCAmelCase						:				Optional[Any]		  =			AutoTokenizer.from_pretrained(_a				)
           self.assertEqual(len(_a				)       ,    len(_a				)				)
           UpperCAmelCase						:				Union[str, Any]		  =			json.loads(new_tokenizer._tokenizer.to_str()				)
           self.assertNotIn("huggingface"       ,    json_tokenizer["model"]["vocab"]				)

 def   _UpperCamelCase    (			self					: Optional[int]				)    ->			List[str]:
      # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
      UpperCAmelCase						:				List[Any]		  =			"hf-internal-testing/test-two-tokenizers"

      # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
      UpperCAmelCase						:				List[str]		  =			AutoTokenizer.from_pretrained(_a				)
      self.assertEqual(len(_a				)       ,    2_8_9_9_7				)
      UpperCAmelCase						:				List[str]		  =			json.loads(tokenizer._tokenizer.to_str()				)
      self.assertIn("huggingface"       ,    json_tokenizer["model"]["vocab"]				)

      # Testing an older version by monkey-patching the version in the module it's used.
      import transformers as old_transformers

      UpperCAmelCase						:				List[str]		  =			"3.0.0"
      UpperCAmelCase						:				str		  =			old_transformers.models.auto.AutoTokenizer.from_pretrained(_a				)
      self.assertEqual(len(_a				)       ,    2_8_9_9_6				)
      UpperCAmelCase						:				Dict		  =			json.loads(old_tokenizer._tokenizer.to_str()				)
      self.assertNotIn("huggingface"       ,    json_tokenizer["model"]["vocab"]				)




@require_tokenizers
class    A_   ( unittest.TestCase     ):







 """simple docstring"""





 def   _UpperCamelCase    (			self					: str				)    ->			Union[str, Any]:
      # See https://github.com/huggingface/transformers/pull/12550
      # and https://github.com/huggingface/tokenizers/issues/537
      UpperCAmelCase						:				Optional[int]		  =			PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"				)
      UpperCAmelCase						:				int		  =			"The Matrix is a 1999 science fiction action film."

      with concurrent.futures.ThreadPoolExecutor() as executor:
           UpperCAmelCase						:				Optional[int]		  =			[executor.submit(self.fetch       ,    _a       ,    _a				) for i in range(1_0				)]
           UpperCAmelCase						:				Optional[Any]		  =			[future.result() for future in futures]
           self.assertEqual(_a       ,    [[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0				)]				)

 def   _UpperCamelCase    (			self					: Dict       ,    _a					: Tuple       ,    _a					: Optional[int]				)    ->			Optional[int]:
      return tokenizer.encode(_a       ,    truncation="longest_first"       ,    padding="longest"				)







from __future__ import annotations

from collections.abc import Callable
from typing import Generic, TypeVar

_snake_case   = TypeVar("T")
_snake_case   = TypeVar("U")




class    A_   ( Generic[T, U]     ):







 """simple docstring"""





 def __init__(			self					: List[Any]       ,    _a					: T | None       ,    _a					: U | None				)    ->			Dict:
      UpperCAmelCase						:				str		  =			key
      UpperCAmelCase						:				Dict		  =			val
      UpperCAmelCase						:				int		  =			0
      UpperCAmelCase						:				DoubleLinkedListNode[T, U] | None		  =			None
      UpperCAmelCase						:				DoubleLinkedListNode[T, U] | None		  =			None

 def __repr__(			self					: List[str]				)    ->			str:
      return "Node: key: {}, val: {}, freq: {}, has next: {}, has prev: {}".format(
          self.key       ,    self.val       ,    self.freq       ,    self.next is not None       ,    self.prev is not None				)




class    A_   ( Generic[T, U]     ):







 """simple docstring"""





 def __init__(			self					: Any				)    ->			None:
      UpperCAmelCase						:				DoubleLinkedListNode[T, U]		  =			DoubleLinkedListNode(_a       ,    _a				)
      UpperCAmelCase						:				DoubleLinkedListNode[T, U]		  =			DoubleLinkedListNode(_a       ,    _a				)
      UpperCAmelCase  , UpperCAmelCase						:				Optional[int]		  =			self.rear, self.head
 def __repr__(			self					: int				)    ->			str:
      UpperCAmelCase						:				Union[str, Any]		  =			["DoubleLinkedList"]
      UpperCAmelCase						:				int		  =			self.head
      while node.next is not None:
           rep.append(str(_a				)				)
           UpperCAmelCase						:				Union[str, Any]		  =			node.next
      rep.append(str(self.rear				)				)
      return ",\n    ".join(_a				)
 def   _UpperCamelCase    (			self					: List[Any]       ,    _a					: DoubleLinkedListNode[T, U]				)    ->			None:
      UpperCAmelCase						:				str		  =			self.rear.prev

      # All nodes other than self.head are guaranteed to have non-None previous
      assert previous is not None

      UpperCAmelCase						:				Any		  =			node
      UpperCAmelCase						:				Optional[int]		  =			previous
      UpperCAmelCase						:				Optional[int]		  =			node
      UpperCAmelCase						:				Union[str, Any]		  =			self.rear
      node.freq += 1
      self._position_node(_a				)
 def   _UpperCamelCase    (			self					: Any       ,    _a					: DoubleLinkedListNode[T, U]				)    ->			None:

      while node.prev is not None and node.prev.freq > node.freq:
           # swap node with previous node
           UpperCAmelCase						:				List[str]		  =			node.prev

           UpperCAmelCase						:				Any		  =			previous_node.prev
           UpperCAmelCase						:				int		  =			node.prev
           UpperCAmelCase						:				Tuple		  =			previous_node
           UpperCAmelCase						:				Optional[int]		  =			node

 def   _UpperCamelCase    (			self					: str       ,    _a					: DoubleLinkedListNode[T, U]				)    ->			DoubleLinkedListNode[T, U] | None:

      if node.prev is None or node.next is None:
           return None

      UpperCAmelCase						:				Dict		  =			node.next
      UpperCAmelCase						:				Optional[Any]		  =			node.prev
      UpperCAmelCase						:				List[Any]		  =			None
      UpperCAmelCase						:				int		  =			None
      return node




class    A_   ( Generic[T, U]     ):







 """simple docstring"""





 _UpperCamelCase				: dict[Callable[[T], U], LFUCache[T, U]]      =					{}
 def __init__(			self					: List[Any]       ,    _a					: int				)    ->			int:
      UpperCAmelCase						:				DoubleLinkedList[T, U]		  =			DoubleLinkedList()
      UpperCAmelCase						:				Any		  =			capacity
      UpperCAmelCase						:				Union[str, Any]		  =			0
      UpperCAmelCase						:				Any		  =			0
      UpperCAmelCase						:				Dict		  =			0
      UpperCAmelCase						:				dict[T, DoubleLinkedListNode[T, U]]		  =			{}
 def __repr__(			self					: Union[str, Any]				)    ->			str:
      return (
          f'CacheInfo(hits={self.hits}, misses={self.miss}, '
          f'capacity={self.capacity}, current_size={self.num_keys})'
      )
 def __contains__(			self					: Optional[int]       ,    _a					: T				)    ->			bool:
      return key in self.cache
 def   _UpperCamelCase    (			self					: Optional[int]       ,    _a					: T				)    ->			U | None:

      if key in self.cache:
           self.hits += 1
           UpperCAmelCase						:				DoubleLinkedListNode[T, U]		  =			self.cache[key]
           UpperCAmelCase						:				Optional[int]		  =			self.list.remove(self.cache[key]				)
           assert node == value_node

           # node is guaranteed not None because it is in self.cache
           assert node is not None
           self.list.add(_a				)
           return node.val
      self.miss += 1
      return None
 def   _UpperCamelCase    (			self					: str       ,    _a					: T       ,    _a					: U				)    ->			None:

      if key not in self.cache:
           if self.num_keys >= self.capacity:
                # delete first node when over capacity
                UpperCAmelCase						:				Any		  =			self.list.head.next

                # guaranteed to have a non-None first node when num_keys > 0
                # explain to type checker via assertions
                assert first_node is not None
                assert first_node.key is not None
                assert self.list.remove(_a				) is not None
                # first_node guaranteed to be in list

                del self.cache[first_node.key]
                self.num_keys -= 1
           UpperCAmelCase						:				Optional[Any]		  =			DoubleLinkedListNode(_a       ,    _a				)
           self.list.add(self.cache[key]				)
           self.num_keys += 1

      else:
           UpperCAmelCase						:				Any		  =			self.list.remove(self.cache[key]				)
           assert node is not None  # node guaranteed to be in list
           UpperCAmelCase						:				str		  =			value
           self.list.add(_a				)

 @classmethod
 def   _UpperCamelCase    (			cls					: type[LFUCache[T, U]]       ,    _a					: int = 1_2_8				)    ->			Callable[[Callable[[T], U]], Callable[..., U]]:

      def cache_decorator_inner(_a					: Callable[[T], U]				) -> Callable[..., U]:
           def cache_decorator_wrapper(*_a					: T				) -> U:
                if func not in cls.decorator_function_to_instance_map:
                     UpperCAmelCase						:				List[Any]		  =			LFUCache(_a				)

                UpperCAmelCase						:				List[Any]		  =			cls.decorator_function_to_instance_map[func].get(args[0]				)
                if result is None:
                     UpperCAmelCase						:				List[Any]		  =			func(*_a				)
                     cls.decorator_function_to_instance_map[func].put(args[0]       ,    _a				)
                return result

           def cache_info() -> LFUCache[T, U]:
                return cls.decorator_function_to_instance_map[func]

           setattr(_a       ,    "cache_info"       ,    _a				)  # noqa: B010

           return cache_decorator_wrapper

      return cache_decorator_inner


if __name__ == "__main__":
      import doctest

      doctest.testmod()



