




import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class       __lowerCAmelCase				(							lowerCAmelCase					,       unittest.TestCase):
       _a           =  GPTaTokenizer
       _a           =  GPTaTokenizerFast
       _a           =  True
       _a           =  {'''add_prefix_space''': True}
       _a           =  False






       def     SCREAMING_SNAKE_CASE			(      self:    List[Any] ):
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         lowercase			:Optional[Any]							=  [
             "l",
             "o",
             "w",
             "e",
             "r",
             "s",
             "t",
             "i",
             "d",
             "n",
             "\u0120",
             "\u0120l",
             "\u0120n",
             "\u0120lo",
             "\u0120low",
             "er",
             "\u0120lowest",
             "\u0120newer",
             "\u0120wider",
             "<unk>",
             "<|endoftext|>",
         ]
         lowercase			:Optional[Any]							=  dict(zip(_lowerCAmelCase				,    range(len(_lowerCAmelCase ) ) ) )
         lowercase			:Dict							=  ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
         lowercase			:Optional[Any]							=  {"unk_token": "<unk>"}

         lowercase			:Optional[Any]							=  os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES["vocab_file"] )
         lowercase			:Optional[int]							=  os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES["merges_file"] )
         with open(self.vocab_file				,    "w"				,    encoding="utf-8" ) as fp:
           fp.write(json.dumps(_lowerCAmelCase ) + "\n" )
         with open(self.merges_file				,    "w"				,    encoding="utf-8" ) as fp:
           fp.write("\n".join(_lowerCAmelCase ) )






       def     SCREAMING_SNAKE_CASE			(      self:    Tuple				,    **_lowerCAmelCase:    List[str] ):
         kwargs.update(self.special_tokens_map )
         return GPTaTokenizer.from_pretrained(self.tmpdirname				,    **_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    List[str]				,    **_lowerCAmelCase:    Optional[int] ):
         kwargs.update(self.special_tokens_map )
         return GPTaTokenizerFast.from_pretrained(self.tmpdirname				,    **_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Tuple				,    _lowerCAmelCase:    Optional[int] ):
         lowercase			:List[str]							=  "lower newer"
         lowercase			:List[Any]							=  "lower newer"
         return input_text, output_text






       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         lowercase			:Dict							=  GPTaTokenizer(self.vocab_file				,    self.merges_file				,    **self.special_tokens_map )
         lowercase			:Any							=  "lower newer"
         lowercase			:List[str]							=  ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
         lowercase			:Optional[Any]							=  tokenizer.tokenize(_lowerCAmelCase				,    add_prefix_space=_lowerCAmelCase )
         self.assertListEqual(_lowerCAmelCase				,    _lowerCAmelCase )

         lowercase			:str							=  tokens + [tokenizer.unk_token]
         lowercase			:List[Any]							=  [14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase )				,    _lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         if not self.test_rust_tokenizer:
           return

         lowercase			:Union[str, Any]							=  self.get_tokenizer()
         lowercase			:Any							=  self.get_rust_tokenizer(add_prefix_space=_lowerCAmelCase )

         lowercase			:Dict							=  "lower newer"

         # Testing tokenization
         lowercase			:Union[str, Any]							=  tokenizer.tokenize(_lowerCAmelCase				,    add_prefix_space=_lowerCAmelCase )
         lowercase			:Tuple							=  rust_tokenizer.tokenize(_lowerCAmelCase )
         self.assertListEqual(_lowerCAmelCase				,    _lowerCAmelCase )

         # Testing conversion to ids without special tokens
         lowercase			:Dict							=  tokenizer.encode(_lowerCAmelCase				,    add_special_tokens=_lowerCAmelCase				,    add_prefix_space=_lowerCAmelCase )
         lowercase			:List[Any]							=  rust_tokenizer.encode(_lowerCAmelCase				,    add_special_tokens=_lowerCAmelCase )
         self.assertListEqual(_lowerCAmelCase				,    _lowerCAmelCase )

         # Testing conversion to ids with special tokens
         lowercase			:Dict							=  self.get_rust_tokenizer(add_prefix_space=_lowerCAmelCase )
         lowercase			:Union[str, Any]							=  tokenizer.encode(_lowerCAmelCase				,    add_prefix_space=_lowerCAmelCase )
         lowercase			:List[str]							=  rust_tokenizer.encode(_lowerCAmelCase )
         self.assertListEqual(_lowerCAmelCase				,    _lowerCAmelCase )

         # Testing the unknown token
         lowercase			:List[str]							=  tokens + [rust_tokenizer.unk_token]
         lowercase			:Dict							=  [14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_lowerCAmelCase )				,    _lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Dict				,    *_lowerCAmelCase:    Union[str, Any]				,    **_lowerCAmelCase:    List[Any] ):
         # It's very difficult to mix/test pretokenization with byte-level
         # And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
         pass






       def     SCREAMING_SNAKE_CASE			(      self:    Any				,    _lowerCAmelCase:    Union[str, Any]=15 ):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
             lowercase			:str							=  self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase				,    **_lowerCAmelCase )

             # Simple input
             lowercase			:Optional[Any]							=  "This is a simple input"
             lowercase			:Union[str, Any]							=  ["This is a simple input 1", "This is a simple input 2"]
             lowercase			:Optional[Any]							=  ("This is a simple input", "This is a pair")
             lowercase			:Union[str, Any]							=  [
                 ("This is a simple input 1", "This is a simple input 2"),
                 ("This is a simple pair 1", "This is a simple pair 2"),
             ]

             # Simple input tests
             self.assertRaises(_lowerCAmelCase				,    tokenizer_r.encode				,    _lowerCAmelCase				,    max_length=_lowerCAmelCase				,    padding="max_length" )

             # Simple input
             self.assertRaises(_lowerCAmelCase				,    tokenizer_r.encode_plus				,    _lowerCAmelCase				,    max_length=_lowerCAmelCase				,    padding="max_length" )

             # Simple input
             self.assertRaises(
                 _lowerCAmelCase				,    tokenizer_r.batch_encode_plus				,    _lowerCAmelCase				,    max_length=_lowerCAmelCase				,    padding="max_length"				,    )

             # Pair input
             self.assertRaises(_lowerCAmelCase				,    tokenizer_r.encode				,    _lowerCAmelCase				,    max_length=_lowerCAmelCase				,    padding="max_length" )

             # Pair input
             self.assertRaises(_lowerCAmelCase				,    tokenizer_r.encode_plus				,    _lowerCAmelCase				,    max_length=_lowerCAmelCase				,    padding="max_length" )

             # Pair input
             self.assertRaises(
                 _lowerCAmelCase				,    tokenizer_r.batch_encode_plus				,    _lowerCAmelCase				,    max_length=_lowerCAmelCase				,    padding="max_length"				,    )






       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any] ):
         lowercase			:int							=  GPTaTokenizer.from_pretrained(self.tmpdirname				,    pad_token="<pad>" )

         # Simple input
         lowercase			:str							=  "This is a simple input"
         lowercase			:List[str]							=  ["This is a simple input looooooooong", "This is a simple input"]
         lowercase			:str							=  ("This is a simple input", "This is a pair")
         lowercase			:Optional[Any]							=  [
             ("This is a simple input loooooong", "This is a simple input"),
             ("This is a simple pair loooooong", "This is a simple pair"),
         ]

         lowercase			:List[Any]							=  tokenizer.pad_token_id

         lowercase			:Any							=  tokenizer(_lowerCAmelCase				,    padding="max_length"				,    max_length=30				,    return_tensors="np" )
         lowercase			:int							=  tokenizer(_lowerCAmelCase				,    padding=_lowerCAmelCase				,    truncate=_lowerCAmelCase				,    return_tensors="np" )
         lowercase			:Optional[Any]							=  tokenizer(*_lowerCAmelCase				,    padding="max_length"				,    max_length=60				,    return_tensors="np" )
         lowercase			:Tuple							=  tokenizer(_lowerCAmelCase				,    padding=_lowerCAmelCase				,    truncate=_lowerCAmelCase				,    return_tensors="np" )

         # s
         # test single string max_length padding
         self.assertEqual(out_s["input_ids"].shape[-1]				,    30 )
         self.assertTrue(pad_token_id in out_s["input_ids"] )
         self.assertTrue(0 in out_s["attention_mask"] )

         # s2
         # test automatic padding
         self.assertEqual(out_sa["input_ids"].shape[-1]				,    33 )
         # long slice doesn't have padding
         self.assertFalse(pad_token_id in out_sa["input_ids"][0] )
         self.assertFalse(0 in out_sa["attention_mask"][0] )
         # short slice does have padding
         self.assertTrue(pad_token_id in out_sa["input_ids"][1] )
         self.assertTrue(0 in out_sa["attention_mask"][1] )

         # p
         # test single pair max_length padding
         self.assertEqual(out_p["input_ids"].shape[-1]				,    60 )
         self.assertTrue(pad_token_id in out_p["input_ids"] )
         self.assertTrue(0 in out_p["attention_mask"] )

         # p2
         # test automatic padding pair
         self.assertEqual(out_pa["input_ids"].shape[-1]				,    52 )
         # long slice pair doesn't have padding
         self.assertFalse(pad_token_id in out_pa["input_ids"][0] )
         self.assertFalse(0 in out_pa["attention_mask"][0] )
         # short slice pair does have padding
         self.assertTrue(pad_token_id in out_pa["input_ids"][1] )
         self.assertTrue(0 in out_pa["attention_mask"][1] )






       def     SCREAMING_SNAKE_CASE			(      self:    Tuple ):
         lowercase			:List[str]							=  "$$$"
         lowercase			:Any							=  GPTaTokenizer.from_pretrained(self.tmpdirname				,    bos_token=_lowerCAmelCase				,    add_bos_token=_lowerCAmelCase )

         lowercase			:List[Any]							=  "This is a simple input"
         lowercase			:Dict							=  ["This is a simple input 1", "This is a simple input 2"]

         lowercase			:Union[str, Any]							=  tokenizer.bos_token_id

         lowercase			:Tuple							=  tokenizer(_lowerCAmelCase )
         lowercase			:Optional[Any]							=  tokenizer(_lowerCAmelCase )

         self.assertEqual(out_s.input_ids[0]				,    _lowerCAmelCase )
         self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids ) )

         lowercase			:int							=  tokenizer.decode(out_s.input_ids )
         lowercase			:Any							=  tokenizer.batch_decode(out_sa.input_ids )

         self.assertEqual(decode_s.split()[0]				,    _lowerCAmelCase )
         self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa ) )






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int] ):
         pass






       def     SCREAMING_SNAKE_CASE			(      self:    int ):
         # TODO: change to self.get_tokenizers() when the fast version is implemented
         lowercase			:List[Any]							=  [self.get_tokenizer(do_lower_case=_lowerCAmelCase				,    add_bos_token=_lowerCAmelCase )]
         for tokenizer in tokenizers:
           with self.subTest(F"{tokenizer.__class__.__name__}" ):
             lowercase			:List[Any]							=  "Encode this."
             lowercase			:List[Any]							=  "This one too please."
             lowercase			:Tuple							=  tokenizer.encode(_lowerCAmelCase				,    add_special_tokens=_lowerCAmelCase )
             encoded_sequence += tokenizer.encode(_lowerCAmelCase				,    add_special_tokens=_lowerCAmelCase )
             lowercase			:Optional[Any]							=  tokenizer.encode_plus(
                 _lowerCAmelCase				,    _lowerCAmelCase				,    add_special_tokens=_lowerCAmelCase				,    return_special_tokens_mask=_lowerCAmelCase				,    )
             lowercase			:Any							=  encoded_sequence_dict["input_ids"]
             lowercase			:List[Any]							=  encoded_sequence_dict["special_tokens_mask"]
             self.assertEqual(len(_lowerCAmelCase )				,    len(_lowerCAmelCase ) )

             lowercase			:Any							=  [
                 (x if not special_tokens_mask[i] else None) for i, x in enumerate(_lowerCAmelCase )
             ]
             lowercase			:int							=  [x for x in filtered_sequence if x is not None]
             self.assertEqual(_lowerCAmelCase				,    _lowerCAmelCase )







@require_tokenizers
class       __lowerCAmelCase				(							unittest.TestCase):






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int] ):
         # More context:
         # https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
         # https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
         # https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

         lowercase			:List[Any]							=  AutoTokenizer.from_pretrained("facebook/opt-350m"				,    from_slow=_lowerCAmelCase )
         lowercase			:Any							=  "A photo of a cat"

         lowercase			:Optional[int]							=  tokenizer.encode(
             _lowerCAmelCase				,    )
         self.assertEqual(_lowerCAmelCase				,    [2, 2_50, 13_45, 9, 10, 47_58] )
         tokenizer.save_pretrained("test_opt" )

         lowercase			:List[Any]							=  AutoTokenizer.from_pretrained("./test_opt" )
         lowercase			:List[Any]							=  tokenizer.encode(
             _lowerCAmelCase				,    )
         self.assertEqual(_lowerCAmelCase				,    [2, 2_50, 13_45, 9, 10, 47_58] )






       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         lowercase			:Optional[int]							=  AutoTokenizer.from_pretrained("facebook/opt-350m"				,    use_slow=_lowerCAmelCase )
         lowercase			:Any							=  "A photo of a cat"

         lowercase			:int							=  tokenizer.encode(
             _lowerCAmelCase				,    )
         # Same as above
         self.assertEqual(_lowerCAmelCase				,    [2, 2_50, 13_45, 9, 10, 47_58] )






       @unittest.skip("This test is failing because of a bug in the fast tokenizer" )
       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int] ):
         lowercase			:Any							=  AutoTokenizer.from_pretrained("facebook/opt-350m"				,    from_slow=_lowerCAmelCase )

         lowercase			:List[Any]							=  "bos"
         lowercase			:List[str]							=  tokenizer.get_vocab()["bos"]

         lowercase			:Tuple							=  "A photo of a cat"
         lowercase			:Dict							=  tokenizer.encode(
             _lowerCAmelCase				,    )
         # We changed the bos token
         self.assertEqual(_lowerCAmelCase				,    [3_19_57, 2_50, 13_45, 9, 10, 47_58] )
         tokenizer.save_pretrained("./tok" )
         lowercase			:Optional[Any]							=  AutoTokenizer.from_pretrained("./tok" )
         self.assertTrue(tokenizer.is_fast )
         lowercase			:Dict							=  tokenizer.encode(
             _lowerCAmelCase				,    )
         self.assertEqual(_lowerCAmelCase				,    [3_19_57, 2_50, 13_45, 9, 10, 47_58] )









import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase		:  Dict	=   logging.get_logger(__name__)

_UpperCAmelCase		:  Optional[int]	=   "▁"

_UpperCAmelCase		:  int	=   {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
}

_UpperCAmelCase		:  List[Any]	=   {
    "vocab_file": {
        "facebook/s2t-small-librispeech-asr": (
            "https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json"
        ),
    },
    "spm_file": {
        "facebook/s2t-small-librispeech-asr": (
            "https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model"
        )
    },
}

_UpperCAmelCase		:  str	=   {
    "facebook/s2t-small-librispeech-asr": 1024,
}

_UpperCAmelCase		:  List[str]	=   ["pt", "fr", "ru", "nl", "ro", "it", "es", "de"]

_UpperCAmelCase		:  Tuple	=   {"mustc": MUSTC_LANGS}




class       __lowerCAmelCase				(							lowerCAmelCase):
       _a           =  VOCAB_FILES_NAMES
       _a           =  PRETRAINED_VOCAB_FILES_MAP
       _a           =  MAX_MODEL_INPUT_SIZES
       _a           =  ['''input_ids''', '''attention_mask''']

       _a           =  []






       def __init__(      self:    Dict				,    _lowerCAmelCase:    Optional[Any]				,    _lowerCAmelCase:    Optional[Any]				,    _lowerCAmelCase:    List[Any]="<s>"				,    _lowerCAmelCase:    List[str]="</s>"				,    _lowerCAmelCase:    Optional[int]="<pad>"				,    _lowerCAmelCase:    List[str]="<unk>"				,    _lowerCAmelCase:    Dict=False				,    _lowerCAmelCase:    Optional[Any]=False				,    _lowerCAmelCase:    Dict=None				,    _lowerCAmelCase:    Dict=None				,    _lowerCAmelCase:    Optional[Dict[str, Any]] = None				,    **_lowerCAmelCase:    Tuple				,    ):
         lowercase			:str							=  {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=_lowerCAmelCase				,    eos_token=_lowerCAmelCase				,    unk_token=_lowerCAmelCase				,    pad_token=_lowerCAmelCase				,    do_upper_case=_lowerCAmelCase				,    do_lower_case=_lowerCAmelCase				,    tgt_lang=_lowerCAmelCase				,    lang_codes=_lowerCAmelCase				,    sp_model_kwargs=self.sp_model_kwargs				,    **_lowerCAmelCase				,    )
         lowercase			:List[str]							=  do_upper_case
         lowercase			:List[str]							=  do_lower_case

         lowercase			:str							=  load_json(_lowerCAmelCase )
         lowercase			:List[Any]							=  {v: k for k, v in self.encoder.items()}
         lowercase			:str							=  spm_file
         lowercase			:Any							=  load_spm(_lowerCAmelCase				,    self.sp_model_kwargs )

         if lang_codes is not None:
           lowercase			:List[str]							=  lang_codes
           lowercase			:Tuple							=  LANGUAGES[lang_codes]
           lowercase			:Any							=  [F"<lang:{lang}>" for lang in self.langs]
           lowercase			:Any							=  {lang: self.sp_model.PieceToId(F"<lang:{lang}>" ) for lang in self.langs}

           lowercase			:Optional[Any]							=  self.lang_tokens
           lowercase			:List[str]							=  tgt_lang if tgt_lang is not None else self.langs[0]

           self.set_tgt_lang_special_tokens(self._tgt_lang )
         else:
           lowercase			:Tuple							=  {}






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    Tuple ):
         return len(self.encoder )






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any] ):
         return self._tgt_lang






       @tgt_lang.setter
       def     SCREAMING_SNAKE_CASE			(      self:    List[Any]				,    _lowerCAmelCase:    Optional[Any] ):
         lowercase			:Any							=  new_tgt_lang
         self.set_tgt_lang_special_tokens(_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int]				,    _lowerCAmelCase:    str ):
         lowercase			:Any							=  self.lang_code_to_id[tgt_lang]
         lowercase			:List[Any]							=  [lang_code_id]






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any]				,    _lowerCAmelCase:    str ):
         return self.sp_model.encode(_lowerCAmelCase				,    out_type=_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Dict				,    _lowerCAmelCase:    Tuple ):
         return self.encoder.get(_lowerCAmelCase				,    self.encoder[self.unk_token] )






       def     SCREAMING_SNAKE_CASE			(      self:    List[str]				,    _lowerCAmelCase:    int ):
         return self.decoder.get(_lowerCAmelCase				,    self.unk_token )






       def     SCREAMING_SNAKE_CASE			(      self:    List[Any]				,    _lowerCAmelCase:    List[str] ):
         lowercase			:int							=  []
         lowercase			:Any							=  ""
         for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
             lowercase			:Dict							=  self.sp_model.decode(_lowerCAmelCase )
             out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
             lowercase			:Any							=  []
           else:
             current_sub_tokens.append(_lowerCAmelCase )
         lowercase			:Dict							=  self.sp_model.decode(_lowerCAmelCase )
         out_string += decoded.upper() if self.do_upper_case else decoded
         return out_string.strip()






       def     SCREAMING_SNAKE_CASE			(      self:    Any				,    _lowerCAmelCase:    int				,    _lowerCAmelCase:    str=None ):
         if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + [self.eos_token_id]
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]






       def     SCREAMING_SNAKE_CASE			(      self:    int				,    _lowerCAmelCase:    List[int]				,    _lowerCAmelCase:    Optional[List[int]] = None				,    _lowerCAmelCase:    bool = False ):

         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=_lowerCAmelCase				,    token_ids_a=_lowerCAmelCase				,    already_has_special_tokens=_lowerCAmelCase )

         lowercase			:int							=  [1] * len(self.prefix_tokens )
         lowercase			:List[str]							=  [1]
         if token_ids_a is None:
           return prefix_ones + ([0] * len(_lowerCAmelCase )) + suffix_ones
         return prefix_ones + ([0] * len(_lowerCAmelCase )) + ([0] * len(_lowerCAmelCase )) + suffix_ones






       def     SCREAMING_SNAKE_CASE			(      self:    Dict ):
         lowercase			:int							=  self.encoder.copy()
         vocab.update(self.added_tokens_encoder )
         return vocab






       def __getstate__(      self:    Optional[Any] ):
         lowercase			:Union[str, Any]							=  self.__dict__.copy()
         lowercase			:Dict							=  None
         return state






       def __setstate__(      self:    Union[str, Any]				,    _lowerCAmelCase:    Dict ):
         lowercase			:List[Any]							=  d

         # for backward compatibility
         if not hasattr(self				,    "sp_model_kwargs" ):
           lowercase			:List[Any]							=  {}

         lowercase			:Union[str, Any]							=  load_spm(self.spm_file				,    self.sp_model_kwargs )






       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any]				,    _lowerCAmelCase:    str				,    _lowerCAmelCase:    Optional[str] = None ):
         lowercase			:List[str]							=  Path(_lowerCAmelCase )
         assert save_dir.is_dir(), F"{save_directory} should be a directory"
         lowercase			:List[Any]							=  save_dir / (
             (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
         )
         lowercase			:Any							=  save_dir / (
             (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
         )

         save_json(self.encoder				,    _lowerCAmelCase )

         if os.path.abspath(self.spm_file ) != os.path.abspath(_lowerCAmelCase ) and os.path.isfile(self.spm_file ):
           copyfile(self.spm_file				,    _lowerCAmelCase )
         elif not os.path.isfile(self.spm_file ):
           with open(_lowerCAmelCase				,    "wb" ) as fi:
             lowercase			:int							=  self.sp_model.serialized_model_proto()
             fi.write(_lowerCAmelCase )

         return (str(_lowerCAmelCase ), str(_lowerCAmelCase ))







def 					UpperCAmelCase__     (   lowerCamelCase,	lowerCamelCase      ):
  lowercase			:Tuple							=  sentencepiece.SentencePieceProcessor(**lowerCamelCase      )
  spm.Load(str(lowerCamelCase      )      )
  return spm



def 					UpperCAmelCase__     (   lowerCamelCase      ):
  with open(lowerCamelCase,	"r"      ) as f:
    return json.load(lowerCamelCase      )







def 					UpperCAmelCase__     (   lowerCamelCase,	lowerCamelCase      ):
  with open(lowerCamelCase,	"w"      ) as f:
    json.dump(lowerCamelCase,	lowerCamelCase,	indent=2      )



