import copy
import inspect
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import VideoMAEConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch
   from torch import nn

   from transformers import (
       MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
       VideoMAEForPreTraining,
       VideoMAEForVideoClassification,
       VideoMAEModel,
   )
   from transformers.models.videomae.modeling_videomae import VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
   from transformers import VideoMAEImageProcessor







class __snake_case				:




   def __init__(    self			:				Optional[Any]       ,	_lowercase			:				Any       ,	_lowercase			:				List[Any]=13       ,	_lowercase			:				Any=10       ,	_lowercase			:				Optional[Any]=3       ,	_lowercase			:				Union[str, Any]=2       ,	_lowercase			:				Dict=2       ,	_lowercase			:				Tuple=2       ,	_lowercase			:				List[str]=True       ,	_lowercase			:				Optional[int]=True       ,	_lowercase			:				Dict=32       ,	_lowercase			:				Any=5       ,	_lowercase			:				Dict=4       ,	_lowercase			:				Any=37       ,	_lowercase			:				Union[str, Any]="gelu"       ,	_lowercase			:				Dict=0.1       ,	_lowercase			:				Union[str, Any]=0.1       ,	_lowercase			:				Dict=10       ,	_lowercase			:				str=0.02       ,	_lowercase			:				List[Any]=0.9       ,	_lowercase			:				List[Any]=None       ,	):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      parent
     SCREAMING_SNAKE_CASE__										=      batch_size
     SCREAMING_SNAKE_CASE__										=      image_size
     SCREAMING_SNAKE_CASE__										=      num_channels
     SCREAMING_SNAKE_CASE__										=      patch_size
     SCREAMING_SNAKE_CASE__										=      tubelet_size
     SCREAMING_SNAKE_CASE__										=      num_frames
     SCREAMING_SNAKE_CASE__										=      is_training
     SCREAMING_SNAKE_CASE__										=      use_labels
     SCREAMING_SNAKE_CASE__										=      hidden_size
     SCREAMING_SNAKE_CASE__										=      num_hidden_layers
     SCREAMING_SNAKE_CASE__										=      num_attention_heads
     SCREAMING_SNAKE_CASE__										=      intermediate_size
     SCREAMING_SNAKE_CASE__										=      hidden_act
     SCREAMING_SNAKE_CASE__										=      hidden_dropout_prob
     SCREAMING_SNAKE_CASE__										=      attention_probs_dropout_prob
     SCREAMING_SNAKE_CASE__										=      type_sequence_label_size
     SCREAMING_SNAKE_CASE__										=      initializer_range
     SCREAMING_SNAKE_CASE__										=      mask_ratio
     SCREAMING_SNAKE_CASE__										=      scope

     # in VideoMAE, the number of tokens equals num_frames/tubelet_size * num_patches per frame
     SCREAMING_SNAKE_CASE__										=      (image_size // patch_size) ** 2
     SCREAMING_SNAKE_CASE__										=      (num_frames // tubelet_size) * self.num_patches_per_frame

     # use this variable to define bool_masked_pos
     SCREAMING_SNAKE_CASE__										=      int(mask_ratio * self.seq_length      )




   def 	__a						(    self			:				Dict      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      floats_tensor(
         [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size]      )

     SCREAMING_SNAKE_CASE__										=      None
     if self.use_labels:
       SCREAMING_SNAKE_CASE__										=      ids_tensor([self.batch_size]       ,	self.type_sequence_label_size      )

     SCREAMING_SNAKE_CASE__										=      self.get_config()

     return config, pixel_values, labels




   def 	__a						(    self			:				Union[str, Any]      ):



     """simple docstring"""






     return VideoMAEConfig(
         image_size=self.image_size       ,	patch_size=self.patch_size       ,	num_channels=self.num_channels       ,	num_frames=self.num_frames       ,	tubelet_size=self.tubelet_size       ,	hidden_size=self.hidden_size       ,	num_hidden_layers=self.num_hidden_layers       ,	num_attention_heads=self.num_attention_heads       ,	intermediate_size=self.intermediate_size       ,	hidden_act=self.hidden_act       ,	hidden_dropout_prob=self.hidden_dropout_prob       ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,	is_decoder=_lowercase       ,	initializer_range=self.initializer_range       ,	)




   def 	__a						(    self			:				Dict       ,	_lowercase			:				Tuple       ,	_lowercase			:				Optional[Any]       ,	_lowercase			:				Any      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      VideoMAEModel(config=_lowercase      )
     model.to(_lowercase      )
     model.eval()
     SCREAMING_SNAKE_CASE__										=      model(_lowercase      )
     self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)      )




   def 	__a						(    self			:				Any       ,	_lowercase			:				str       ,	_lowercase			:				Optional[int]       ,	_lowercase			:				str      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      VideoMAEForPreTraining(_lowercase      )
     model.to(_lowercase      )
     model.eval()
     # important: each video needs to have the same number of masked patches
     # hence we define a single mask, which we then repeat for each example in the batch
     SCREAMING_SNAKE_CASE__										=      torch.ones((self.num_masks,)      )
     SCREAMING_SNAKE_CASE__										=      torch.cat([mask, torch.zeros(self.seq_length - mask.size(0      )      )]      )
     SCREAMING_SNAKE_CASE__										=      mask.expand(self.batch_size       ,	-1      ).bool()

     SCREAMING_SNAKE_CASE__										=      model(_lowercase       ,	_lowercase      )
     # model only returns predictions for masked patches
     SCREAMING_SNAKE_CASE__										=      mask.sum().item()
     SCREAMING_SNAKE_CASE__										=      3 * self.tubelet_size * self.patch_size**2
     self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, num_masked_patches, decoder_num_labels)      )




   def 	__a						(    self			:				Optional[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.prepare_config_and_inputs()
     SCREAMING_SNAKE_CASE__										=      config_and_inputs
     SCREAMING_SNAKE_CASE__										=      {'''pixel_values''': pixel_values}
     return config, inputs_dict







@require_torch
class __snake_case				(						lowerCAmelCase_   ,		lowerCAmelCase_   ,		unittest.TestCase		):
   lowerCAmelCase_														=							(
       (VideoMAEModel, VideoMAEForPreTraining, VideoMAEForVideoClassification) if is_torch_available() else ()
   )
   lowerCAmelCase_														=							(
       {"feature-extraction": VideoMAEModel, "video-classification": VideoMAEForVideoClassification}
       if is_torch_available()
       else {}
   )

   lowerCAmelCase_														=							False
   lowerCAmelCase_														=							False
   lowerCAmelCase_														=							False
   lowerCAmelCase_														=							False




   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      VideoMAEModelTester(self      )
     SCREAMING_SNAKE_CASE__										=      ConfigTester(self       ,	config_class=_lowercase       ,	has_text_modality=_lowercase       ,	hidden_size=37      )




   def 	__a						(    self			:				Any       ,	_lowercase			:				List[Any]       ,	_lowercase			:				Optional[Any]       ,	_lowercase			:				List[str]=False      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      copy.deepcopy(_lowercase      )

     if model_class == VideoMAEForPreTraining:
       # important: each video needs to have the same number of masked patches
       # hence we define a single mask, which we then repeat for each example in the batch
       SCREAMING_SNAKE_CASE__										=      torch.ones((self.model_tester.num_masks,)      )
       SCREAMING_SNAKE_CASE__										=      torch.cat([mask, torch.zeros(self.model_tester.seq_length - mask.size(0      )      )]      )
       SCREAMING_SNAKE_CASE__										=      mask.expand(self.model_tester.batch_size       ,	-1      ).bool()
       SCREAMING_SNAKE_CASE__										=      bool_masked_pos.to(_lowercase      )

     if return_labels:
       if model_class in [
           *get_values(_lowercase      ),
       ]:
         SCREAMING_SNAKE_CASE__										=      torch.zeros(
             self.model_tester.batch_size       ,	dtype=torch.long       ,	device=_lowercase      )

     return inputs_dict




   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     self.config_tester.run_common_tests()




   @unittest.skip(reason="""VideoMAE does not use inputs_embeds"""      )
   def 	__a						(    self			:				List[str]      ):



     """simple docstring"""






     pass




   def 	__a						(    self			:				Union[str, Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
       SCREAMING_SNAKE_CASE__										=      model_class(_lowercase      )
       self.assertIsInstance(model.get_input_embeddings()       ,	(nn.Module)      )
       SCREAMING_SNAKE_CASE__										=      model.get_output_embeddings()
       self.assertTrue(x is None or isinstance(_lowercase       ,	nn.Linear      )      )




   def 	__a						(    self			:				Optional[int]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
       SCREAMING_SNAKE_CASE__										=      model_class(_lowercase      )
       SCREAMING_SNAKE_CASE__										=      inspect.signature(model.forward      )
       # signature.parameters is an OrderedDict => so arg_names order is deterministic
       SCREAMING_SNAKE_CASE__										=      [*signature.parameters.keys()]

       SCREAMING_SNAKE_CASE__										=      ['''pixel_values''']
       self.assertListEqual(arg_names[:1]       ,	_lowercase      )




   def 	__a						(    self			:				Tuple      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*_lowercase      )




   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_pretraining(*_lowercase      )




   @slow
   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     for model_name in VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       SCREAMING_SNAKE_CASE__										=      VideoMAEModel.from_pretrained(_lowercase      )
       self.assertIsNotNone(_lowercase      )




   def 	__a						(    self			:				List[str]      ):



     """simple docstring"""






     if not self.has_attentions:
       pass

     else:
       SCREAMING_SNAKE_CASE__										=      self.model_tester.prepare_config_and_inputs_for_common()
       SCREAMING_SNAKE_CASE__										=      True

       for model_class in self.all_model_classes:
         SCREAMING_SNAKE_CASE__										=      self.model_tester.seq_length - self.model_tester.num_masks
         SCREAMING_SNAKE_CASE__										=      (
             num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length
         )

         SCREAMING_SNAKE_CASE__										=      True
         SCREAMING_SNAKE_CASE__										=      False
         SCREAMING_SNAKE_CASE__										=      True
         SCREAMING_SNAKE_CASE__										=      model_class(_lowercase      )
         model.to(_lowercase      )
         model.eval()
         with torch.no_grad():
           SCREAMING_SNAKE_CASE__										=      model(**self._prepare_for_class(_lowercase       ,	_lowercase      )      )
         SCREAMING_SNAKE_CASE__										=      outputs.attentions
         self.assertEqual(len(_lowercase      )       ,	self.model_tester.num_hidden_layers      )

         # check that output_attentions also work using config
         del inputs_dict["output_attentions"]
         SCREAMING_SNAKE_CASE__										=      True
         SCREAMING_SNAKE_CASE__										=      model_class(_lowercase      )
         model.to(_lowercase      )
         model.eval()
         with torch.no_grad():
           SCREAMING_SNAKE_CASE__										=      model(**self._prepare_for_class(_lowercase       ,	_lowercase      )      )
         SCREAMING_SNAKE_CASE__										=      outputs.attentions
         self.assertEqual(len(_lowercase      )       ,	self.model_tester.num_hidden_layers      )

         self.assertListEqual(
             list(attentions[0].shape[-3:]      )       ,	[self.model_tester.num_attention_heads, seq_len, seq_len]       ,	)
         SCREAMING_SNAKE_CASE__										=      len(_lowercase      )

         # Check attention is always last and order is fine
         SCREAMING_SNAKE_CASE__										=      True
         SCREAMING_SNAKE_CASE__										=      True
         SCREAMING_SNAKE_CASE__										=      model_class(_lowercase      )
         model.to(_lowercase      )
         model.eval()
         with torch.no_grad():
           SCREAMING_SNAKE_CASE__										=      model(**self._prepare_for_class(_lowercase       ,	_lowercase      )      )

         self.assertEqual(out_len + 1       ,	len(_lowercase      )      )

         SCREAMING_SNAKE_CASE__										=      outputs.attentions

         self.assertEqual(len(_lowercase      )       ,	self.model_tester.num_hidden_layers      )
         self.assertListEqual(
             list(self_attentions[0].shape[-3:]      )       ,	[self.model_tester.num_attention_heads, seq_len, seq_len]       ,	)




   def 	__a						(    self			:				str      ):



     """simple docstring"""






     def check_hidden_states_output(_lowercase			:				Optional[Any]       ,	_lowercase			:				List[str]       ,	_lowercase			:				Optional[Any]      ):
       SCREAMING_SNAKE_CASE__										=      model_class(_lowercase      )
       model.to(_lowercase      )
       model.eval()

       with torch.no_grad():
         SCREAMING_SNAKE_CASE__										=      model(**self._prepare_for_class(_lowercase       ,	_lowercase      )      )

       SCREAMING_SNAKE_CASE__										=      outputs.hidden_states
       SCREAMING_SNAKE_CASE__										=      self.model_tester.num_hidden_layers + 1
       self.assertEqual(len(_lowercase      )       ,	_lowercase      )

       SCREAMING_SNAKE_CASE__										=      self.model_tester.seq_length - self.model_tester.num_masks
       SCREAMING_SNAKE_CASE__										=      num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length

       self.assertListEqual(
           list(hidden_states[0].shape[-2:]      )       ,	[seq_length, self.model_tester.hidden_size]       ,	)

     SCREAMING_SNAKE_CASE__										=      self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
       SCREAMING_SNAKE_CASE__										=      True
       check_hidden_states_output(_lowercase       ,	_lowercase       ,	_lowercase      )

       # check that output_hidden_states also work using config
       del inputs_dict["output_hidden_states"]
       SCREAMING_SNAKE_CASE__										=      True

       check_hidden_states_output(_lowercase       ,	_lowercase       ,	_lowercase      )




   @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""      )
   def 	__a						(    self			:				Optional[int]      ):



     """simple docstring"""






     pass





def 	__SCREAMING_SNAKE_CASE  (					)							->					Union[str, Any]:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__										=      hf_hub_download(
      repo_id="""hf-internal-testing/spaghetti-video""" ,     filename="""eating_spaghetti.npy""" ,     repo_type="""dataset"""	)
  SCREAMING_SNAKE_CASE__										=      np.load(__lowerCamelCase	)
  return list(__lowerCamelCase	)







@require_torch
@require_vision
class __snake_case				(						unittest.TestCase		):




   @cached_property
   def 	__a						(    self			:				List[str]      ):



     """simple docstring"""






     return (
         VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]       ,	image_std=[0.5, 0.5, 0.5]      )
         if is_vision_available()
         else None
     )




   @slow
   def 	__a						(    self			:				Optional[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      VideoMAEForVideoClassification.from_pretrained("""MCG-NJU/videomae-base-finetuned-kinetics"""      ).to(
         _lowercase      )

     SCREAMING_SNAKE_CASE__										=      self.default_image_processor
     SCREAMING_SNAKE_CASE__										=      prepare_video()
     SCREAMING_SNAKE_CASE__										=      image_processor(_lowercase       ,	return_tensors="""pt"""      ).to(_lowercase      )

     # forward pass
     with torch.no_grad():
       SCREAMING_SNAKE_CASE__										=      model(**_lowercase      )

     # verify the logits
     SCREAMING_SNAKE_CASE__										=      torch.Size((1, 4_00)      )
     self.assertEqual(outputs.logits.shape       ,	_lowercase      )

     SCREAMING_SNAKE_CASE__										=      torch.tensor([0.36_69, -0.06_88, -0.24_21]      ).to(_lowercase      )

     self.assertTrue(torch.allclose(outputs.logits[0, :3]       ,	_lowercase       ,	atol=1E-4      )      )




   @slow
   def 	__a						(    self			:				Any      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      VideoMAEForPreTraining.from_pretrained("""MCG-NJU/videomae-base-short"""      ).to(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      self.default_image_processor
     SCREAMING_SNAKE_CASE__										=      prepare_video()
     SCREAMING_SNAKE_CASE__										=      image_processor(_lowercase       ,	return_tensors="""pt"""      ).to(_lowercase      )

     # add boolean mask, indicating which patches to mask
     SCREAMING_SNAKE_CASE__										=      hf_hub_download(repo_id="""hf-internal-testing/bool-masked-pos"""       ,	filename="""bool_masked_pos.pt"""      )
     SCREAMING_SNAKE_CASE__										=      torch.load(_lowercase      )

     # forward pass
     with torch.no_grad():
       SCREAMING_SNAKE_CASE__										=      model(**_lowercase      )

     # verify the logits
     SCREAMING_SNAKE_CASE__										=      torch.Size([1, 14_08, 15_36]      )
     SCREAMING_SNAKE_CASE__										=      torch.tensor(
         [[0.79_94, 0.96_12, 0.85_08], [0.74_01, 0.89_58, 0.83_02], [0.58_62, 0.74_68, 0.73_25]]       ,	device=_lowercase      )
     self.assertEqual(outputs.logits.shape       ,	_lowercase      )
     self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]       ,	_lowercase       ,	atol=1E-4      )      )

     # verify the loss (`config.norm_pix_loss` = `True`)
     SCREAMING_SNAKE_CASE__										=      torch.tensor([0.51_42]       ,	device=_lowercase      )
     self.assertTrue(torch.allclose(outputs.loss       ,	_lowercase       ,	atol=1E-4      )      )

     # verify the loss (`config.norm_pix_loss` = `False`)
     SCREAMING_SNAKE_CASE__										=      VideoMAEForPreTraining.from_pretrained("""MCG-NJU/videomae-base-short"""       ,	norm_pix_loss=_lowercase      ).to(
         _lowercase      )

     with torch.no_grad():
       SCREAMING_SNAKE_CASE__										=      model(**_lowercase      )

     SCREAMING_SNAKE_CASE__										=      torch.tensor(torch.tensor([0.64_69]      )       ,	device=_lowercase      )
     self.assertTrue(torch.allclose(outputs.loss       ,	_lowercase       ,	atol=1E-4      )      )





"""simple docstring"""






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
		import torch


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


_lowercase : Optional[int]      						=						logging.get_logger(__name__)
_lowercase : Optional[int]      						=						{"vocab_file": "spiece.model"}

_lowercase : str      						=						{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

_lowercase : Optional[Any]      						=						{
    "AI-Sweden/gpt-sw3-126m": 2_0_4_8,
    "AI-Sweden/gpt-sw3-350m": 2_0_4_8,
    "AI-Sweden/gpt-sw3-1.6b": 2_0_4_8,
    "AI-Sweden/gpt-sw3-6.7b": 2_0_4_8,
    "AI-Sweden/gpt-sw3-20b": 2_0_4_8,
}

class        __SCREAMING_SNAKE_CASE	(      lowerCAmelCase_   ):


					'''simple docstring'''





					_a   			=							VOCAB_FILES_NAMES
					_a   			=							PRETRAINED_VOCAB_FILES_MAP
					_a   			=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_a   			=							['input_ids', 'attention_mask']



					def __init__(       self							:  Dict,  lowerCamelCase							:  Tuple,  lowerCamelCase							:  List[str]=False,  lowerCamelCase							:  str=False,  lowerCamelCase							:  List[Any]=False,  lowerCamelCase							:  List[str]=None,  lowerCamelCase							:  Optional[Any]=None,  lowerCamelCase							:  List[str]=None,  lowerCamelCase							:  Optional[int]=None,  lowerCamelCase							:  Optional[Dict[str, Any]] = None,  **lowerCamelCase							:  Dict,  )->					None:
								lowerCamelCase__		:		Any			={} if sp_model_kwargs is None else sp_model_kwargs

								lowerCamelCase__		:		Optional[int]			=kwargs.get('''name_or_path'''     )
								if name_or_path is None:
											logger.warning(
											    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
											    ''' you are testing the model, this can safely be ignored'''     )
											lowerCamelCase__		:		List[Any]			='''None'''

								# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
								lowerCamelCase__		:		Tuple			='''<|endoftext|>''' if eos_token is None else eos_token
								lowerCamelCase__		:		Dict			='''<unk>''' if unk_token is None else unk_token
								if "gpt-sw3-7b" in name_or_path:
											lowerCamelCase__		:		int			=unk_token if pad_token is None else pad_token
											lowerCamelCase__		:		str			=eos_token if bos_token is None else bos_token
								else:
											lowerCamelCase__		:		Any			='''<pad>''' if pad_token is None else pad_token
											lowerCamelCase__		:		str			='''<s>''' if bos_token is None else bos_token

								super().__init__(
								    do_lower_case=lowerCamelCase,  remove_space=lowerCamelCase,  keep_accents=lowerCamelCase,  bos_token=lowerCamelCase,  eos_token=lowerCamelCase,  unk_token=lowerCamelCase,  pad_token=lowerCamelCase,  sp_model_kwargs=self.sp_model_kwargs,  **lowerCamelCase,  )

								lowerCamelCase__		:		Union[str, Any]			=do_lower_case
								lowerCamelCase__		:		Union[str, Any]			=remove_space
								lowerCamelCase__		:		int			=keep_accents
								lowerCamelCase__		:		Tuple			=vocab_file

								lowerCamelCase__		:		List[str]			=spm.SentencePieceProcessor(**self.sp_model_kwargs     )
								self.sp_model.Load(lowerCamelCase     )

								# Used for whitespace normalization in input texts
								# fmt : off
								lowerCamelCase__		:		List[Any]			={''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
								# fmt : on

								# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
								lowerCamelCase__		:		int			=re.compile(
								    F'''[{"".join(map(lowerCamelCase,  list(range(0,  9     )     ) + list(range(11,  32     )     ) + list(range(127,  160     )     ) + [160, 173, 8203]     )     )}]'''     )



					def __getstate__(       self							:  Any     )->					Optional[Any]:
								lowerCamelCase__		:		List[str]			=self.__dict__.copy()
								lowerCamelCase__		:		List[Any]			=None
								return state



					def __setstate__(       self							:  Optional[Any],  lowerCamelCase							:  Dict     )->					int:
								lowerCamelCase__		:		List[str]			=d

								# for backward compatibility
								if not hasattr(self,  '''sp_model_kwargs'''     ):
											lowerCamelCase__		:		Union[str, Any]			={}

								lowerCamelCase__		:		str			=spm.SentencePieceProcessor(**self.sp_model_kwargs     )
								self.sp_model.Load(self.vocab_file     )



					@property
					# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
					def 		snake_case       (       self							:  List[str]     )->					int:
								return len(self.sp_model     )



					def 		snake_case       (       self							:  List[str],  lowerCamelCase							:  str     )->					str:
								lowerCamelCase__		:		Dict			=self.non_printing_characters_re.sub('''''',  lowerCamelCase     )

								# Normalize whitespaces
								lowerCamelCase__		:		List[Any]			=''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]     )

								# NFC Unicode normalization
								lowerCamelCase__		:		Union[str, Any]			=unicodedata.normalize('''NFC''',  lowerCamelCase     )
								return text



					def 		snake_case       (       self							:  int,  lowerCamelCase							:  str,  **lowerCamelCase							:  str     )->					List[str]:
								lowerCamelCase__		:		int			=self.preprocess_text(lowerCamelCase     )
								return self.sp_model.encode(lowerCamelCase,  out_type=lowerCamelCase     )



					def 		snake_case       (       self							:  Any,  lowerCamelCase							:  str     )->					int:
								return self.sp_model.PieceToId(lowerCamelCase     )



					def 		snake_case       (       self							:  Optional[int],  lowerCamelCase							:  int     )->					str:
								return self.sp_model.IdToPiece(lowerCamelCase     )



					@staticmethod
					def 		snake_case       (       lowerCamelCase							:  str     )->					str:
								return out_string



					def 		snake_case       (       self							:  List[Any],  lowerCamelCase							:  List[str]     )->					str:
								lowerCamelCase__		:		Dict			=[]
								lowerCamelCase__		:		int			=''''''
								lowerCamelCase__		:		Any			=False
								for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
														# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
														if not prev_is_special:
																	out_string += " "

														out_string += self.sp_model.decode(lowerCamelCase     ) + token
														lowerCamelCase__		:		Tuple			=True
														lowerCamelCase__		:		Tuple			=[]
											else:
														current_sub_tokens.append(lowerCamelCase     )
														lowerCamelCase__		:		Optional[Any]			=False
								out_string += self.sp_model.decode(lowerCamelCase     )

								return out_string



					def 		snake_case       (       self							:  Union[str, Any]     )->					Dict[str, int]:
								lowerCamelCase__		:		Tuple			={self.convert_ids_to_tokens(lowerCamelCase     ): i for i in range(self.vocab_size     )}
								vocab.update(self.added_tokens_encoder     )
								return vocab



					def 		snake_case       (       self							:  Optional[Any],  lowerCamelCase							:  str,  lowerCamelCase							:  Optional[str] = None     )->					Tuple[str]:
								if not os.path.isdir(lowerCamelCase     ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
											return
								lowerCamelCase__		:		List[str]			=os.path.join(
								    lowerCamelCase,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )

								if os.path.abspath(self.vocab_file     ) != os.path.abspath(lowerCamelCase     ) and os.path.isfile(self.vocab_file     ):
											copyfile(self.vocab_file,  lowerCamelCase     )
								elif not os.path.isfile(self.vocab_file     ):
											with open(lowerCamelCase,  '''wb'''     ) as fi:
														lowerCamelCase__		:		Dict			=self.sp_model.serialized_model_proto()
														fi.write(lowerCamelCase     )

								return (out_vocab_file,)



					def 		snake_case       (       self							:  int,  lowerCamelCase							:  Union[str, List[str]],  lowerCamelCase							:  Union[str, bool] = False     )->					Union[List[int], List[List[int]], "torch.Tensor"]:

								if isinstance(lowerCamelCase,  lowerCamelCase     ):
											lowerCamelCase__		:		str			=self.preprocess_text(lowerCamelCase     )
											lowerCamelCase__		:		Union[str, Any]			=self.sp_model.encode(lowerCamelCase     )
								else:
											lowerCamelCase__		:		int			=[self.preprocess_text(lowerCamelCase     ) for t in text]
											lowerCamelCase__		:		Tuple			=self.sp_model.encode(lowerCamelCase     )

								if return_tensors is True or return_tensors == "pt":
											lowerCamelCase__		:		Union[str, Any]			=torch.tensor(lowerCamelCase     )

								return token_ids



					def 		snake_case       (       self							:  int,  lowerCamelCase							:  Union[int, List[int]]     )->					str:
								return self.sp_model.decode(lowerCamelCase     )




					def 		snake_case       (       self							:  Tuple,  lowerCamelCase							:  "Conversation"     )->					List[int]:
								lowerCamelCase__		:		Any			=[F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
								lowerCamelCase__		:		str			=(
								    F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(lowerCamelCase     ) + F'''{self.bos_token}Bot:'''
								)
								return self.encode(text=lowerCamelCase     )




