




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast




class       __lowerCAmelCase				(							unittest.TestCase):






       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         # A mock response for an HTTP head request to emulate server down
         lowercase			:Optional[int]							=  mock.Mock()
         lowercase			:List[str]							=  5_00
         lowercase			:int							=  {}
         lowercase			:Any							=  HTTPError
         lowercase			:int							=  {}

         # Download this model to make sure it's in the cache.
         lowercase			:str							=  BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert" )

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch("requests.Session.request"				,    return_value=_lowerCAmelCase ) as mock_head:
           lowercase			:int							=  BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert" )
           # This check we did call the fake head request
           mock_head.assert_called()






       @require_tokenizers
       def     SCREAMING_SNAKE_CASE			(      self:    Any ):
         # A mock response for an HTTP head request to emulate server down
         lowercase			:int							=  mock.Mock()
         lowercase			:str							=  5_00
         lowercase			:List[Any]							=  {}
         lowercase			:Tuple							=  HTTPError
         lowercase			:Union[str, Any]							=  {}

         # Download this model to make sure it's in the cache.
         lowercase			:str							=  GPTaTokenizerFast.from_pretrained("gpt2" )

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch("requests.Session.request"				,    return_value=_lowerCAmelCase ) as mock_head:
           lowercase			:Dict							=  GPTaTokenizerFast.from_pretrained("gpt2" )
           # This check we did call the fake head request
           mock_head.assert_called()






       def     SCREAMING_SNAKE_CASE			(      self:    Any ):
         # This test is for deprecated behavior and can be removed in v5
         try:
           lowercase			:Optional[Any]							=  tempfile.mktemp()
           with open(_lowerCAmelCase				,    "wb" ) as f:
             http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"				,    _lowerCAmelCase )

           lowercase			:Union[str, Any]							=  AlbertTokenizer.from_pretrained(_lowerCAmelCase )
         finally:
           os.remove(_lowerCAmelCase )

         # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
         # the current folder and have the right name.
         if os.path.isfile("tokenizer.json" ):
           # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
           return
         try:
           with open("tokenizer.json"				,    "wb" ) as f:
             http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"				,    _lowerCAmelCase )
           lowercase			:Optional[Any]							=  AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2" )
           # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
           self.assertEqual(tokenizer.vocab_size				,    10_00 )
           # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

         finally:
           os.remove("tokenizer.json" )






       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any] ):
         # This test is for deprecated behavior and can be removed in v5
         lowercase			:Dict							=  AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model" )







@is_staging_test
class       __lowerCAmelCase				(							unittest.TestCase):
       _a           =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']






       @classmethod
       def     SCREAMING_SNAKE_CASE			(      cls:    List[str] ):
         lowercase			:List[Any]							=  TOKEN
         HfFolder.save_token(_lowerCAmelCase )






       @classmethod
       def     SCREAMING_SNAKE_CASE			(      cls:    List[Any] ):
         try:
           delete_repo(token=cls._token				,    repo_id="test-tokenizer" )
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token				,    repo_id="valid_org/test-tokenizer-org" )
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token				,    repo_id="test-dynamic-tokenizer" )
         except HTTPError:
           pass






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any] ):
         with tempfile.TemporaryDirectory() as tmp_dir:
           lowercase			:Optional[int]							=  os.path.join(_lowerCAmelCase				,    "vocab.txt" )
           with open(_lowerCAmelCase				,    "w"				,    encoding="utf-8" ) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens] ) )
           lowercase			:Any							=  BertTokenizer(_lowerCAmelCase )

         tokenizer.push_to_hub("test-tokenizer"				,    use_auth_token=self._token )
         lowercase			:int							=  BertTokenizer.from_pretrained(F"{USER}/test-tokenizer" )
         self.assertDictEqual(new_tokenizer.vocab				,    tokenizer.vocab )

         # Reset repo
         delete_repo(token=self._token				,    repo_id="test-tokenizer" )

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(_lowerCAmelCase				,    repo_id="test-tokenizer"				,    push_to_hub=_lowerCAmelCase				,    use_auth_token=self._token )

         lowercase			:int							=  BertTokenizer.from_pretrained(F"{USER}/test-tokenizer" )
         self.assertDictEqual(new_tokenizer.vocab				,    tokenizer.vocab )






       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any] ):
         with tempfile.TemporaryDirectory() as tmp_dir:
           lowercase			:List[str]							=  os.path.join(_lowerCAmelCase				,    "vocab.txt" )
           with open(_lowerCAmelCase				,    "w"				,    encoding="utf-8" ) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens] ) )
           lowercase			:Optional[int]							=  BertTokenizer(_lowerCAmelCase )

         tokenizer.push_to_hub("valid_org/test-tokenizer-org"				,    use_auth_token=self._token )
         lowercase			:List[Any]							=  BertTokenizer.from_pretrained("valid_org/test-tokenizer-org" )
         self.assertDictEqual(new_tokenizer.vocab				,    tokenizer.vocab )

         # Reset repo
         delete_repo(token=self._token				,    repo_id="valid_org/test-tokenizer-org" )

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(
               _lowerCAmelCase				,    repo_id="valid_org/test-tokenizer-org"				,    push_to_hub=_lowerCAmelCase				,    use_auth_token=self._token )

         lowercase			:int							=  BertTokenizer.from_pretrained("valid_org/test-tokenizer-org" )
         self.assertDictEqual(new_tokenizer.vocab				,    tokenizer.vocab )






       @require_tokenizers
       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         CustomTokenizer.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           lowercase			:Union[str, Any]							=  os.path.join(_lowerCAmelCase				,    "vocab.txt" )
           with open(_lowerCAmelCase				,    "w"				,    encoding="utf-8" ) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens] ) )
           lowercase			:Any							=  CustomTokenizer(_lowerCAmelCase )

         # No fast custom tokenizer
         tokenizer.push_to_hub("test-dynamic-tokenizer"				,    use_auth_token=self._token )

         lowercase			:str							=  AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"				,    trust_remote_code=_lowerCAmelCase )
         # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__				,    "CustomTokenizer" )

         # Fast and slow custom tokenizer
         CustomTokenizerFast.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           lowercase			:Tuple							=  os.path.join(_lowerCAmelCase				,    "vocab.txt" )
           with open(_lowerCAmelCase				,    "w"				,    encoding="utf-8" ) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens] ) )

           lowercase			:Optional[int]							=  BertTokenizerFast.from_pretrained(_lowerCAmelCase )
           bert_tokenizer.save_pretrained(_lowerCAmelCase )
           lowercase			:List[Any]							=  CustomTokenizerFast.from_pretrained(_lowerCAmelCase )

         tokenizer.push_to_hub("test-dynamic-tokenizer"				,    use_auth_token=self._token )

         lowercase			:Union[str, Any]							=  AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"				,    trust_remote_code=_lowerCAmelCase )
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__				,    "CustomTokenizerFast" )
         lowercase			:Optional[int]							=  AutoTokenizer.from_pretrained(
             F"{USER}/test-dynamic-tokenizer"				,    use_fast=_lowerCAmelCase				,    trust_remote_code=_lowerCAmelCase )
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__				,    "CustomTokenizer" )







class       __lowerCAmelCase				(							unittest.TestCase):






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any] ):
         lowercase			:Optional[int]							=  Trie()
         trie.add("Hello 友達" )
         self.assertEqual(trie.data				,    {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}} )
         trie.add("Hello" )
         trie.data
         self.assertEqual(trie.data				,    {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}} )






       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         lowercase			:List[str]							=  Trie()
         self.assertEqual(trie.split("[CLS] This is a extra_id_100" )				,    ["[CLS] This is a extra_id_100"] )
         trie.add("[CLS]" )
         trie.add("extra_id_1" )
         trie.add("extra_id_100" )
         self.assertEqual(trie.split("[CLS] This is a extra_id_100" )				,    ["[CLS]", " This is a ", "extra_id_100"] )






       def     SCREAMING_SNAKE_CASE			(      self:    Tuple ):
         lowercase			:Union[str, Any]							=  Trie()
         trie.add("A" )
         self.assertEqual(trie.split("ABC" )				,    ["A", "BC"] )
         self.assertEqual(trie.split("BCA" )				,    ["BC", "A"] )






       def     SCREAMING_SNAKE_CASE			(      self:    Tuple ):
         lowercase			:int							=  Trie()
         trie.add("TOKEN]" )
         trie.add("[SPECIAL_TOKEN]" )
         self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]" )				,    ["This is something ", "[SPECIAL_TOKEN]"] )






       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         lowercase			:Dict							=  Trie()
         trie.add("A" )
         trie.add("P" )
         trie.add("[SPECIAL_TOKEN]" )
         self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]" )				,    ["This is something ", "[SPECIAL_TOKEN]"] )






       def     SCREAMING_SNAKE_CASE			(      self:    Any ):
         lowercase			:List[Any]							=  Trie()
         trie.add("AB" )
         trie.add("B" )
         trie.add("C" )
         self.assertEqual(trie.split("ABC" )				,    ["AB", "C"] )






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any] ):
         lowercase			:int							=  Trie()
         trie.add("ABC" )
         trie.add("B" )
         trie.add("CD" )
         self.assertEqual(trie.split("ABCD" )				,    ["ABC", "D"] )






       def     SCREAMING_SNAKE_CASE			(      self:    int ):
         # Even if the offsets are wrong, we necessarily output correct string
         # parts.
         lowercase			:Dict							=  Trie()
         lowercase			:str							=  trie.cut_text("ABC"				,    [0, 0, 2, 1, 2, 3] )
         self.assertEqual(_lowerCAmelCase				,    ["AB", "C"] )









import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase		:  Dict	=   logging.get_logger(__name__)

_UpperCAmelCase		:  Optional[int]	=   "▁"

_UpperCAmelCase		:  int	=   {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
}

_UpperCAmelCase		:  List[Any]	=   {
    "vocab_file": {
        "facebook/s2t-small-librispeech-asr": (
            "https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json"
        ),
    },
    "spm_file": {
        "facebook/s2t-small-librispeech-asr": (
            "https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model"
        )
    },
}

_UpperCAmelCase		:  str	=   {
    "facebook/s2t-small-librispeech-asr": 1024,
}

_UpperCAmelCase		:  List[str]	=   ["pt", "fr", "ru", "nl", "ro", "it", "es", "de"]

_UpperCAmelCase		:  Tuple	=   {"mustc": MUSTC_LANGS}




class       __lowerCAmelCase				(							lowerCAmelCase):
       _a           =  VOCAB_FILES_NAMES
       _a           =  PRETRAINED_VOCAB_FILES_MAP
       _a           =  MAX_MODEL_INPUT_SIZES
       _a           =  ['''input_ids''', '''attention_mask''']

       _a           =  []






       def __init__(      self:    Dict				,    _lowerCAmelCase:    Optional[Any]				,    _lowerCAmelCase:    Optional[Any]				,    _lowerCAmelCase:    List[Any]="<s>"				,    _lowerCAmelCase:    List[str]="</s>"				,    _lowerCAmelCase:    Optional[int]="<pad>"				,    _lowerCAmelCase:    List[str]="<unk>"				,    _lowerCAmelCase:    Dict=False				,    _lowerCAmelCase:    Optional[Any]=False				,    _lowerCAmelCase:    Dict=None				,    _lowerCAmelCase:    Dict=None				,    _lowerCAmelCase:    Optional[Dict[str, Any]] = None				,    **_lowerCAmelCase:    Tuple				,    ):
         lowercase			:str							=  {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=_lowerCAmelCase				,    eos_token=_lowerCAmelCase				,    unk_token=_lowerCAmelCase				,    pad_token=_lowerCAmelCase				,    do_upper_case=_lowerCAmelCase				,    do_lower_case=_lowerCAmelCase				,    tgt_lang=_lowerCAmelCase				,    lang_codes=_lowerCAmelCase				,    sp_model_kwargs=self.sp_model_kwargs				,    **_lowerCAmelCase				,    )
         lowercase			:List[str]							=  do_upper_case
         lowercase			:List[str]							=  do_lower_case

         lowercase			:str							=  load_json(_lowerCAmelCase )
         lowercase			:List[Any]							=  {v: k for k, v in self.encoder.items()}
         lowercase			:str							=  spm_file
         lowercase			:Any							=  load_spm(_lowerCAmelCase				,    self.sp_model_kwargs )

         if lang_codes is not None:
           lowercase			:List[str]							=  lang_codes
           lowercase			:Tuple							=  LANGUAGES[lang_codes]
           lowercase			:Any							=  [F"<lang:{lang}>" for lang in self.langs]
           lowercase			:Any							=  {lang: self.sp_model.PieceToId(F"<lang:{lang}>" ) for lang in self.langs}

           lowercase			:Optional[Any]							=  self.lang_tokens
           lowercase			:List[str]							=  tgt_lang if tgt_lang is not None else self.langs[0]

           self.set_tgt_lang_special_tokens(self._tgt_lang )
         else:
           lowercase			:Tuple							=  {}






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    Tuple ):
         return len(self.encoder )






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any] ):
         return self._tgt_lang






       @tgt_lang.setter
       def     SCREAMING_SNAKE_CASE			(      self:    List[Any]				,    _lowerCAmelCase:    Optional[Any] ):
         lowercase			:Any							=  new_tgt_lang
         self.set_tgt_lang_special_tokens(_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int]				,    _lowerCAmelCase:    str ):
         lowercase			:Any							=  self.lang_code_to_id[tgt_lang]
         lowercase			:List[Any]							=  [lang_code_id]






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any]				,    _lowerCAmelCase:    str ):
         return self.sp_model.encode(_lowerCAmelCase				,    out_type=_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Dict				,    _lowerCAmelCase:    Tuple ):
         return self.encoder.get(_lowerCAmelCase				,    self.encoder[self.unk_token] )






       def     SCREAMING_SNAKE_CASE			(      self:    List[str]				,    _lowerCAmelCase:    int ):
         return self.decoder.get(_lowerCAmelCase				,    self.unk_token )






       def     SCREAMING_SNAKE_CASE			(      self:    List[Any]				,    _lowerCAmelCase:    List[str] ):
         lowercase			:int							=  []
         lowercase			:Any							=  ""
         for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
             lowercase			:Dict							=  self.sp_model.decode(_lowerCAmelCase )
             out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
             lowercase			:Any							=  []
           else:
             current_sub_tokens.append(_lowerCAmelCase )
         lowercase			:Dict							=  self.sp_model.decode(_lowerCAmelCase )
         out_string += decoded.upper() if self.do_upper_case else decoded
         return out_string.strip()






       def     SCREAMING_SNAKE_CASE			(      self:    Any				,    _lowerCAmelCase:    int				,    _lowerCAmelCase:    str=None ):
         if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + [self.eos_token_id]
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]






       def     SCREAMING_SNAKE_CASE			(      self:    int				,    _lowerCAmelCase:    List[int]				,    _lowerCAmelCase:    Optional[List[int]] = None				,    _lowerCAmelCase:    bool = False ):

         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=_lowerCAmelCase				,    token_ids_a=_lowerCAmelCase				,    already_has_special_tokens=_lowerCAmelCase )

         lowercase			:int							=  [1] * len(self.prefix_tokens )
         lowercase			:List[str]							=  [1]
         if token_ids_a is None:
           return prefix_ones + ([0] * len(_lowerCAmelCase )) + suffix_ones
         return prefix_ones + ([0] * len(_lowerCAmelCase )) + ([0] * len(_lowerCAmelCase )) + suffix_ones






       def     SCREAMING_SNAKE_CASE			(      self:    Dict ):
         lowercase			:int							=  self.encoder.copy()
         vocab.update(self.added_tokens_encoder )
         return vocab






       def __getstate__(      self:    Optional[Any] ):
         lowercase			:Union[str, Any]							=  self.__dict__.copy()
         lowercase			:Dict							=  None
         return state






       def __setstate__(      self:    Union[str, Any]				,    _lowerCAmelCase:    Dict ):
         lowercase			:List[Any]							=  d

         # for backward compatibility
         if not hasattr(self				,    "sp_model_kwargs" ):
           lowercase			:List[Any]							=  {}

         lowercase			:Union[str, Any]							=  load_spm(self.spm_file				,    self.sp_model_kwargs )






       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any]				,    _lowerCAmelCase:    str				,    _lowerCAmelCase:    Optional[str] = None ):
         lowercase			:List[str]							=  Path(_lowerCAmelCase )
         assert save_dir.is_dir(), F"{save_directory} should be a directory"
         lowercase			:List[Any]							=  save_dir / (
             (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
         )
         lowercase			:Any							=  save_dir / (
             (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
         )

         save_json(self.encoder				,    _lowerCAmelCase )

         if os.path.abspath(self.spm_file ) != os.path.abspath(_lowerCAmelCase ) and os.path.isfile(self.spm_file ):
           copyfile(self.spm_file				,    _lowerCAmelCase )
         elif not os.path.isfile(self.spm_file ):
           with open(_lowerCAmelCase				,    "wb" ) as fi:
             lowercase			:int							=  self.sp_model.serialized_model_proto()
             fi.write(_lowerCAmelCase )

         return (str(_lowerCAmelCase ), str(_lowerCAmelCase ))







def 					UpperCAmelCase__     (   lowerCamelCase,	lowerCamelCase      ):
  lowercase			:Tuple							=  sentencepiece.SentencePieceProcessor(**lowerCamelCase      )
  spm.Load(str(lowerCamelCase      )      )
  return spm



def 					UpperCAmelCase__     (   lowerCamelCase      ):
  with open(lowerCamelCase,	"r"      ) as f:
    return json.load(lowerCamelCase      )







def 					UpperCAmelCase__     (   lowerCamelCase,	lowerCamelCase      ):
  with open(lowerCamelCase,	"w"      ) as f:
    json.dump(lowerCamelCase,	lowerCamelCase,	indent=2      )



