


'''simple docstring'''

import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast





class      snake_case    (    unittest.TestCase			):





  """simple docstring"""




  def   snake_case		(    self     ):







   """simple docstring"""
   # A mock response for an HTTP head request to emulate server down
   lowerCamelCase_      =  mock.Mock()
   lowerCamelCase_      =  500
   lowerCamelCase_      =  {}
   lowerCamelCase_      =  HTTPError
   lowerCamelCase_      =  {}

   # Download this model to make sure it's in the cache.
   lowerCamelCase_      =  BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"     )

   # Under the mock environment we get a 500 error when trying to reach the tokenizer.
   with mock.patch("requests.Session.request"			,  return_value=_lowerCAmelCase     ) as mock_head:
    lowerCamelCase_      =  BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"     )
    # This check we did call the fake head request
    mock_head.assert_called()






  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""
   # A mock response for an HTTP head request to emulate server down
   lowerCamelCase_      =  mock.Mock()
   lowerCamelCase_      =  500
   lowerCamelCase_      =  {}
   lowerCamelCase_      =  HTTPError
   lowerCamelCase_      =  {}

   # Download this model to make sure it's in the cache.
   lowerCamelCase_      =  GPTaTokenizerFast.from_pretrained("gpt2"     )

   # Under the mock environment we get a 500 error when trying to reach the tokenizer.
   with mock.patch("requests.Session.request"			,  return_value=_lowerCAmelCase     ) as mock_head:
    lowerCamelCase_      =  GPTaTokenizerFast.from_pretrained("gpt2"     )
    # This check we did call the fake head request
    mock_head.assert_called()






  def   snake_case		(    self     ):







   """simple docstring"""
   # This test is for deprecated behavior and can be removed in v5
   try:
    lowerCamelCase_      =  tempfile.mktemp()
    with open(_lowerCAmelCase			,  "wb"     ) as f:
     http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"			,  _lowerCAmelCase     )

    lowerCamelCase_      =  AlbertTokenizer.from_pretrained(_lowerCAmelCase     )
   finally:
    os.remove(_lowerCAmelCase     )

   # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
   # the current folder and have the right name.
   if os.path.isfile("tokenizer.json"     ):
    # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
    return
   try:
    with open("tokenizer.json"			,  "wb"     ) as f:
     http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"			,  _lowerCAmelCase     )
    lowerCamelCase_      =  AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"     )
    # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
    self.assertEqual(tokenizer.vocab_size			,  1000     )
    # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

   finally:
    os.remove("tokenizer.json"     )







  def   snake_case		(    self     ):







   """simple docstring"""
   # This test is for deprecated behavior and can be removed in v5
   lowerCamelCase_      =  AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"     )





@is_staging_test
class      snake_case    (    unittest.TestCase			):





  """simple docstring"""




  _lowerCamelCase		    =  ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]






  @classmethod
  def   snake_case		(    cls     ):







   """simple docstring"""
   lowerCamelCase_      =  TOKEN
   HfFolder.save_token(_lowerCAmelCase     )






  @classmethod
  def   snake_case		(    cls     ):







   """simple docstring"""
   try:
    delete_repo(token=cls._token			,  repo_id="test-tokenizer"     )
   except HTTPError:
    pass

   try:
    delete_repo(token=cls._token			,  repo_id="valid_org/test-tokenizer-org"     )
   except HTTPError:
    pass

   try:
    delete_repo(token=cls._token			,  repo_id="test-dynamic-tokenizer"     )
   except HTTPError:
    pass






  def   snake_case		(    self     ):







   """simple docstring"""
   with tempfile.TemporaryDirectory() as tmp_dir:
    lowerCamelCase_      =  os.path.join(_lowerCAmelCase			,  "vocab.txt"     )
    with open(_lowerCAmelCase			,  "w"			,  encoding="utf-8"     ) as vocab_writer:
     vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]     )     )
    lowerCamelCase_      =  BertTokenizer(_lowerCAmelCase     )

   tokenizer.push_to_hub("test-tokenizer"			,  use_auth_token=self._token     )
   lowerCamelCase_      =  BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''     )
   self.assertDictEqual(new_tokenizer.vocab			,  tokenizer.vocab     )

   # Reset repo
   delete_repo(token=self._token			,  repo_id="test-tokenizer"     )

   # Push to hub via save_pretrained
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(_lowerCAmelCase			,  repo_id="test-tokenizer"			,  push_to_hub=_lowerCAmelCase			,  use_auth_token=self._token     )

   lowerCamelCase_      =  BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''     )
   self.assertDictEqual(new_tokenizer.vocab			,  tokenizer.vocab     )






  def   snake_case		(    self     ):







   """simple docstring"""
   with tempfile.TemporaryDirectory() as tmp_dir:
    lowerCamelCase_      =  os.path.join(_lowerCAmelCase			,  "vocab.txt"     )
    with open(_lowerCAmelCase			,  "w"			,  encoding="utf-8"     ) as vocab_writer:
     vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]     )     )
    lowerCamelCase_      =  BertTokenizer(_lowerCAmelCase     )

   tokenizer.push_to_hub("valid_org/test-tokenizer-org"			,  use_auth_token=self._token     )
   lowerCamelCase_      =  BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"     )
   self.assertDictEqual(new_tokenizer.vocab			,  tokenizer.vocab     )

   # Reset repo
   delete_repo(token=self._token			,  repo_id="valid_org/test-tokenizer-org"     )

   # Push to hub via save_pretrained
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(
        _lowerCAmelCase			,  repo_id="valid_org/test-tokenizer-org"			,  push_to_hub=_lowerCAmelCase			,  use_auth_token=self._token     )

   lowerCamelCase_      =  BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"     )
   self.assertDictEqual(new_tokenizer.vocab			,  tokenizer.vocab     )







  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""
   CustomTokenizer.register_for_auto_class()
   with tempfile.TemporaryDirectory() as tmp_dir:
    lowerCamelCase_      =  os.path.join(_lowerCAmelCase			,  "vocab.txt"     )
    with open(_lowerCAmelCase			,  "w"			,  encoding="utf-8"     ) as vocab_writer:
     vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]     )     )
    lowerCamelCase_      =  CustomTokenizer(_lowerCAmelCase     )

   # No fast custom tokenizer
   tokenizer.push_to_hub("test-dynamic-tokenizer"			,  use_auth_token=self._token     )

   lowerCamelCase_      =  AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''			,  trust_remote_code=_lowerCAmelCase     )
   # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
   self.assertEqual(tokenizer.__class__.__name__			,  "CustomTokenizer"     )

   # Fast and slow custom tokenizer
   CustomTokenizerFast.register_for_auto_class()
   with tempfile.TemporaryDirectory() as tmp_dir:
    lowerCamelCase_      =  os.path.join(_lowerCAmelCase			,  "vocab.txt"     )
    with open(_lowerCAmelCase			,  "w"			,  encoding="utf-8"     ) as vocab_writer:
     vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]     )     )

    lowerCamelCase_      =  BertTokenizerFast.from_pretrained(_lowerCAmelCase     )
    bert_tokenizer.save_pretrained(_lowerCAmelCase     )
    lowerCamelCase_      =  CustomTokenizerFast.from_pretrained(_lowerCAmelCase     )

   tokenizer.push_to_hub("test-dynamic-tokenizer"			,  use_auth_token=self._token     )

   lowerCamelCase_      =  AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''			,  trust_remote_code=_lowerCAmelCase     )
   # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
   self.assertEqual(tokenizer.__class__.__name__			,  "CustomTokenizerFast"     )
   lowerCamelCase_      =  AutoTokenizer.from_pretrained(
       f'''{USER}/test-dynamic-tokenizer'''			,  use_fast=_lowerCAmelCase			,  trust_remote_code=_lowerCAmelCase     )
   # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
   self.assertEqual(tokenizer.__class__.__name__			,  "CustomTokenizer"     )





class      snake_case    (    unittest.TestCase			):





  """simple docstring"""




  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  Trie()
   trie.add("Hello 友達"     )
   self.assertEqual(trie.data			,  {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}     )
   trie.add("Hello"     )
   trie.data
   self.assertEqual(trie.data			,  {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  Trie()
   self.assertEqual(trie.split("[CLS] This is a extra_id_100"     )			,  ["[CLS] This is a extra_id_100"]     )
   trie.add("[CLS]"     )
   trie.add("extra_id_1"     )
   trie.add("extra_id_100"     )
   self.assertEqual(trie.split("[CLS] This is a extra_id_100"     )			,  ["[CLS]", " This is a ", "extra_id_100"]     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  Trie()
   trie.add("A"     )
   self.assertEqual(trie.split("ABC"     )			,  ["A", "BC"]     )
   self.assertEqual(trie.split("BCA"     )			,  ["BC", "A"]     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  Trie()
   trie.add("TOKEN]"     )
   trie.add("[SPECIAL_TOKEN]"     )
   self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"     )			,  ["This is something ", "[SPECIAL_TOKEN]"]     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  Trie()
   trie.add("A"     )
   trie.add("P"     )
   trie.add("[SPECIAL_TOKEN]"     )
   self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"     )			,  ["This is something ", "[SPECIAL_TOKEN]"]     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  Trie()
   trie.add("AB"     )
   trie.add("B"     )
   trie.add("C"     )
   self.assertEqual(trie.split("ABC"     )			,  ["AB", "C"]     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  Trie()
   trie.add("ABC"     )
   trie.add("B"     )
   trie.add("CD"     )
   self.assertEqual(trie.split("ABCD"     )			,  ["ABC", "D"]     )







  def   snake_case		(    self     ):







   """simple docstring"""
   # Even if the offsets are wrong, we necessarily output correct string
   # parts.
   lowerCamelCase_      =  Trie()
   lowerCamelCase_      =  trie.cut_text("ABC"			,  [0, 0, 2, 1, 2, 3]     )
   self.assertEqual(_lowerCAmelCase			,  ["AB", "C"]     )









from typing import Union

import fire
import torch
from tqdm import tqdm



def 					UpperCAmelCase__     (   lowerCamelCase,	lowerCamelCase = "cpu",	lowerCamelCase = None      ):
  lowercase			:Optional[Any]							=  torch.load(lowerCamelCase,	map_location=lowerCamelCase      )
  for k, v in tqdm(state_dict.items()      ):
    if not isinstance(lowerCamelCase,	torch.Tensor      ):
      raise TypeError("FP16 conversion only works on paths that are saved state dicts, like pytorch_model.bin"      )
    lowercase			:List[Any]							=  v.half()
  if save_path is None:  # overwrite src_path
    lowercase			:Optional[Any]							=  src_path
  torch.save(lowerCamelCase,	lowerCamelCase      )


if __name__ == "__main__":
     fire.Fire(convert)



