code: 
  
  
  '''simple docstring'''
  
  import copy
  import inspect
  import unittest
  
  import numpy as np
  from huggingface_hub import hf_hub_download
  
  from transformers import VivitConfig
  from transformers.models.auto import get_values
  from transformers.testing_utils import require_torch, require_vision, slow, torch_device
  from transformers.utils import cached_property, is_torch_available, is_vision_available
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
  
  
  if is_torch_available():
   import torch
   from torch import nn
  
   from transformers import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING, VivitForVideoClassification, VivitModel
   from transformers.models.vivit.modeling_vivit import VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  if is_vision_available():
   from transformers import VivitImageProcessor
  class 						__lowerCAmelCase		:
  
  
  
  
  
        '''simple docstring'''
  
  
  
  
  
  
        def __init__(self					:					int     ,		UpperCamelCase					:					str     ,		UpperCamelCase					:					str=2     ,		UpperCamelCase					:					Union[str, Any]=True     ,		UpperCamelCase					:					Dict=True     ,		UpperCamelCase					:					List[Any]=10     ,		UpperCamelCase					:					Optional[Any]=10     ,		UpperCamelCase					:					Optional[int]=8     ,		UpperCamelCase					:					int=[2, 4, 4]     ,		UpperCamelCase					:					Dict=3     ,		UpperCamelCase					:					Union[str, Any]=768     ,		UpperCamelCase					:					Any=5     ,		UpperCamelCase					:					List[str]=4     ,		UpperCamelCase					:					Optional[int]=37     ,		UpperCamelCase					:					List[Any]="gelu_fast"     ,		UpperCamelCase					:					List[Any]=0.0     ,		UpperCamelCase					:					str=0.0     ,		UpperCamelCase					:					Tuple=0.02     ,		UpperCamelCase					:					Any=1E-06     ,		UpperCamelCase					:					str=True     ,		UpperCamelCase					:					List[str]=None     ,		):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				parent
               lowercase__       =				batch_size
               lowercase__       =				is_training
               lowercase__       =				use_labels
               lowercase__       =				num_labels
               lowercase__       =				image_size
               lowercase__       =				num_frames
               lowercase__       =				tubelet_size
               lowercase__       =				num_channels
               lowercase__       =				hidden_size
               lowercase__       =				num_hidden_layers
               lowercase__       =				num_attention_heads
               lowercase__       =				intermediate_size
               lowercase__       =				hidden_act
               lowercase__       =				hidden_dropout_prob
               lowercase__       =				attention_probs_dropout_prob
               lowercase__       =				initializer_range
               lowercase__       =				layer_norm_eps
               lowercase__       =				qkv_bias
               lowercase__       =				scope
  
               lowercase__       =				(
                   (self.image_size // self.tubelet_size[2])
                   * (self.image_size // self.tubelet_size[1])
                   * (self.num_frames // self.tubelet_size[0])
               ) + 1  # CLS token
  
  
  
        def    UpperCamelCase__	(self					:					int				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				floats_tensor(
                   [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size]				)
  
               lowercase__       =				None
               if self.use_labels:
                      lowercase__       =				ids_tensor([self.batch_size]     ,		self.num_labels				)
  
               lowercase__       =				self.get_config()
  
               return config, pixel_values, labels
  
  
  
        def    UpperCamelCase__	(self					:					List[str]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				VivitConfig(
                   num_frames=self.num_frames     ,		image_size=self.image_size     ,		tubelet_size=self.tubelet_size     ,		num_channels=self.num_channels     ,		hidden_size=self.hidden_size     ,		num_hidden_layers=self.num_hidden_layers     ,		num_attention_heads=self.num_attention_heads     ,		intermediate_size=self.intermediate_size     ,		hidden_act=self.hidden_act     ,		hidden_dropout_prob=self.hidden_dropout_prob     ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,		initializer_range=self.initializer_range     ,		layer_norm_eps=self.layer_norm_eps     ,		qkv_bias=self.qkv_bias     ,		)
               lowercase__       =				self.num_labels
               return config
  
  
  
        def    UpperCamelCase__	(self					:					Optional[int]     ,		UpperCamelCase					:					Optional[Any]     ,		UpperCamelCase					:					List[str]     ,		UpperCamelCase					:					Dict				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				VivitModel(config=UpperCamelCase				)
               model.to(UpperCamelCase				)
               model.eval()
               lowercase__       =				model(UpperCamelCase				)
               self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, self.seq_length, self.hidden_size)				)
  
  
  
        def    UpperCamelCase__	(self					:					int     ,		UpperCamelCase					:					Union[str, Any]     ,		UpperCamelCase					:					Any     ,		UpperCamelCase					:					Optional[Any]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				VivitForVideoClassification(UpperCamelCase				)
               model.to(UpperCamelCase				)
               model.eval()
  
               lowercase__       =				model(UpperCamelCase				)
  
               # verify the logits shape
               lowercase__       =				torch.Size((self.batch_size, self.num_labels)				)
               self.parent.assertEqual(result.logits.shape     ,		UpperCamelCase				)
  
  
  
  
  
  
        def    UpperCamelCase__	(self					:					Dict				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				self.prepare_config_and_inputs()
               lowercase__    ,lowercase__    ,lowercase__       =				config_and_inputs
               lowercase__       =				{'''pixel_values''': pixel_values}
               return config, inputs_dict
  
  
  
  
  
  
  @require_torch
  class 						__lowerCAmelCase		(lowercase_						,  unittest.TestCase  ):
  
  
  
  
  
        '''simple docstring'''
  
  
  
  
  
  
        lowerCAmelCase__      :  List[str]						      =							(VivitModel, VivitForVideoClassification) if is_torch_available() else ()
  
        lowerCAmelCase__      :  Tuple						      =							False
        lowerCAmelCase__      :  Optional[Any]						      =							False
        lowerCAmelCase__      :  str						      =							False
        lowerCAmelCase__      :  Any						      =							False
  
  
  
        def    UpperCamelCase__	(self					:					Dict				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				VivitModelTester(self				)
               lowercase__       =				ConfigTester(self     ,		config_class=UpperCamelCase     ,		has_text_modality=UpperCamelCase     ,		hidden_size=37				)
  
  
  
        def    UpperCamelCase__	(self					:					List[str]     ,		UpperCamelCase					:					List[Any]     ,		UpperCamelCase					:					List[str]     ,		UpperCamelCase					:					Tuple=False				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				copy.deepcopy(UpperCamelCase				)
  
               if return_labels:
                      if model_class in get_values(UpperCamelCase				):
                             lowercase__       =				torch.zeros(
                                 self.model_tester.batch_size     ,		dtype=torch.long     ,		device=UpperCamelCase				)
  
               return inputs_dict
  
  
  
        def    UpperCamelCase__	(self					:					int				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               self.config_tester.run_common_tests()
  
  
  
        @unittest.skip(reason='''Vivit does not use inputs_embeds'''				)
        def    UpperCamelCase__	(self					:					Optional[Any]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               pass
  
  
  
        def    UpperCamelCase__	(self					:					Optional[int]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__    ,lowercase__       =				self.model_tester.prepare_config_and_inputs_for_common()
  
               for model_class in self.all_model_classes:
                      lowercase__       =				model_class(UpperCamelCase				)
                      self.assertIsInstance(model.get_input_embeddings()     ,		(nn.Module)				)
                      lowercase__       =				model.get_output_embeddings()
                      self.assertTrue(x is None or isinstance(UpperCamelCase     ,		nn.Linear				)				)
  
  
  
        def    UpperCamelCase__	(self					:					Optional[int]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__    ,lowercase__       =				self.model_tester.prepare_config_and_inputs_for_common()
  
               for model_class in self.all_model_classes:
                      lowercase__       =				model_class(UpperCamelCase				)
                      lowercase__       =				inspect.signature(model.forward				)
                      # signature.parameters is an OrderedDict => so arg_names order is deterministic
                      lowercase__       =				[*signature.parameters.keys()]
  
                      lowercase__       =				['''pixel_values''', '''head_mask''']
                      self.assertListEqual(arg_names[:2]     ,		UpperCamelCase				)
  
  
  
        def    UpperCamelCase__	(self					:					str				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				self.model_tester.prepare_config_and_inputs()
               self.model_tester.create_and_check_model(*UpperCamelCase				)
  
  
  
        def    UpperCamelCase__	(self					:					List[Any]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				self.model_tester.prepare_config_and_inputs()
               self.model_tester.create_and_check_for_video_classification(*UpperCamelCase				)
  
  
  
        @slow
        def    UpperCamelCase__	(self					:					Union[str, Any]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               for model_name in VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                      lowercase__       =				VivitModel.from_pretrained(UpperCamelCase				)
                      self.assertIsNotNone(UpperCamelCase				)
  
  
  
        def    UpperCamelCase__	(self					:					Tuple				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__    ,lowercase__       =				self.model_tester.prepare_config_and_inputs_for_common()
               lowercase__       =				True
  
               for model_class in self.all_model_classes:
                      lowercase__       =				self.model_tester.seq_length
  
                      lowercase__       =				True
                      lowercase__       =				False
                      lowercase__       =				True
                      lowercase__       =				model_class(UpperCamelCase				)
                      model.to(UpperCamelCase				)
                      model.eval()
                      with torch.no_grad():
                             lowercase__       =				model(**self._prepare_for_class(UpperCamelCase     ,		UpperCamelCase				)				)
                      lowercase__       =				outputs.attentions
                      self.assertEqual(len(UpperCamelCase				)     ,		self.model_tester.num_hidden_layers				)
  
                      # check that output_attentions also work using config
                      del inputs_dict["output_attentions"]
                      lowercase__       =				True
                      lowercase__       =				model_class(UpperCamelCase				)
                      model.to(UpperCamelCase				)
                      model.eval()
                      with torch.no_grad():
                             lowercase__       =				model(**self._prepare_for_class(UpperCamelCase     ,		UpperCamelCase				)				)
                      lowercase__       =				outputs.attentions
                      self.assertEqual(len(UpperCamelCase				)     ,		self.model_tester.num_hidden_layers				)
  
                      self.assertListEqual(
                          list(attentions[0].shape[-3:]				)     ,		[self.model_tester.num_attention_heads, seq_len, seq_len]     ,		)
                      lowercase__       =				len(UpperCamelCase				)
  
                      # Check attention is always last and order is fine
                      lowercase__       =				True
                      lowercase__       =				True
                      lowercase__       =				model_class(UpperCamelCase				)
                      model.to(UpperCamelCase				)
                      model.eval()
                      with torch.no_grad():
                             lowercase__       =				model(**self._prepare_for_class(UpperCamelCase     ,		UpperCamelCase				)				)
  
                      self.assertEqual(out_len + 1     ,		len(UpperCamelCase				)				)
  
                      lowercase__       =				outputs.attentions
  
                      self.assertEqual(len(UpperCamelCase				)     ,		self.model_tester.num_hidden_layers				)
  
                      self.assertListEqual(
                          list(self_attentions[0].shape[-3:]				)     ,		[self.model_tester.num_attention_heads, seq_len, seq_len]     ,		)
  
  
  
        def    UpperCamelCase__	(self					:					List[Any]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               def check_hidden_states_output(UpperCamelCase					:					Dict     ,		UpperCamelCase					:					Optional[int]     ,		UpperCamelCase					:					Any				):
                      lowercase__       =				model_class(UpperCamelCase				)
                      model.to(UpperCamelCase				)
                      model.eval()
  
                      with torch.no_grad():
                             lowercase__       =				model(**self._prepare_for_class(UpperCamelCase     ,		UpperCamelCase				)				)
  
                      lowercase__       =				outputs.hidden_states
                      lowercase__       =				self.model_tester.num_hidden_layers + 1
                      self.assertEqual(len(UpperCamelCase				)     ,		UpperCamelCase				)
  
                      lowercase__       =				self.model_tester.seq_length
  
                      self.assertListEqual(
                          list(hidden_states[0].shape[-2:]				)     ,		[seq_length, self.model_tester.hidden_size]     ,		)
  
               lowercase__    ,lowercase__       =				self.model_tester.prepare_config_and_inputs_for_common()
  
               for model_class in self.all_model_classes:
                      lowercase__       =				True
                      check_hidden_states_output(UpperCamelCase     ,		UpperCamelCase     ,		UpperCamelCase				)
  
                      # check that output_hidden_states also work using config
                      del inputs_dict["output_hidden_states"]
                      lowercase__       =				True
  
                      check_hidden_states_output(UpperCamelCase     ,		UpperCamelCase     ,		UpperCamelCase				)
  
  
  
  
  
  
        @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''				)
        def    UpperCamelCase__	(self					:					str				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               pass
  
  
  
  
  
  
  def _SCREAMING_SNAKE_CASE   () -> List[Any]:
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         lowercase__       =				hf_hub_download(
             repo_id='''hf-internal-testing/spaghetti-video'''						,				filename='''eating_spaghetti_32_frames.npy'''						,				repo_type='''dataset''' )
         lowercase__       =				np.load(A )
         return list(A )
  
  @require_torch
  @require_vision
  class 						__lowerCAmelCase		(unittest.TestCase  ):
  
  
  
  
  
        '''simple docstring'''
  
  
  
  
  
  
        @cached_property
        def    UpperCamelCase__	(self					:					Optional[int]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               return VivitImageProcessor() if is_vision_available() else None
  
  
  
  
  
  
        @slow
        def    UpperCamelCase__	(self					:					int				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				VivitForVideoClassification.from_pretrained('''google/vivit-b-16x2-kinetics400'''				).to(UpperCamelCase				)
  
               lowercase__       =				self.default_image_processor
               lowercase__       =				prepare_video()
               lowercase__       =				image_processor(UpperCamelCase     ,		return_tensors='''pt'''				).to(UpperCamelCase				)
  
               # forward pass
               with torch.no_grad():
                      lowercase__       =				model(**UpperCamelCase				)
  
               # verify the logits
               lowercase__       =				torch.Size((1, 400)				)
               self.assertEqual(outputs.logits.shape     ,		UpperCamelCase				)
  
               # taken from original model
               lowercase__       =				torch.tensor([-1.05_43, 2.07_64, -0.21_04, 0.44_39, -0.96_58]				).to(UpperCamelCase				)
  
               self.assertTrue(torch.allclose(outputs.logits[0, :5]     ,		UpperCamelCase     ,		atol=1E-4				)				)
  
  
  
  
  
style_context: 
  
  
  '''simple docstring'''
  
  import json
  import os
  from functools import lru_cache
  from typing import TYPE_CHECKING, List, Optional, Tuple, Union
  
  import numpy as np
  import regex as re
  
  from ...utils import is_tf_available, is_torch_available, logging
  
  
  if TYPE_CHECKING:
   if is_torch_available():
    import torch
   if is_tf_available():
    import tensorflow as tf
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  
  
  lowerCamelCase							:     Union[str, Any]								=			logging.get_logger(__name__)
  
  lowerCamelCase							:     Tuple								=			{
      'vocab_file': 'vocab.json',
      'merges_file': 'merges.txt',
  }
  
  lowerCamelCase							:     Tuple								=			{
      'vocab_file': {
          'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
      },
      'merges_file': {
          'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
      },
  }
  
  lowerCamelCase							:     Optional[int]								=			{
      'Salesforce/codegen-350M-mono': 2_048,
  }
  
  @lru_cache()
  def _SCREAMING_SNAKE_CASE   () -> Optional[int]:
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         lowercase__       =				(
             list(range(ord('''!''' )						,				ord('''~''' ) + 1 ) ) + list(range(ord('''¡''' )						,				ord('''¬''' ) + 1 ) ) + list(range(ord('''®''' )						,				ord('''ÿ''' ) + 1 ) )
         )
         lowercase__       =				bs[:]
         lowercase__       =				0
         for b in range(2**8 ):
                if b not in bs:
                       bs.append(A )
                       cs.append(2**8 + n )
                       n += 1
         lowercase__       =				[chr(A ) for n in cs]
         return dict(zip(A						,				A ) )
  
  def _SCREAMING_SNAKE_CASE   (A ) -> Tuple:
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         lowercase__       =				set()
         lowercase__       =				word[0]
         for char in word[1:]:
                pairs.add((prev_char, char) )
                lowercase__       =				char
         return pairs
  
  class 						__lowerCAmelCase		(lowercase_  ):
  
  
  
  
  
        '''simple docstring'''
  
  
  
  
  
  
        lowerCAmelCase__      :  int						      =							VOCAB_FILES_NAMES
        lowerCAmelCase__      :  Union[str, Any]						      =							PRETRAINED_VOCAB_FILES_MAP
        lowerCAmelCase__      :  Optional[int]						      =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        lowerCAmelCase__      :  Tuple						      =							["""input_ids""", """attention_mask"""]
  
  
  
        def __init__(self					:					List[Any]     ,		UpperCamelCase					:					List[str]     ,		UpperCamelCase					:					List[Any]     ,		UpperCamelCase					:					Optional[int]="replace"     ,		UpperCamelCase					:					Optional[Any]="<|endoftext|>"     ,		UpperCamelCase					:					str="<|endoftext|>"     ,		UpperCamelCase					:					Optional[int]="<|endoftext|>"     ,		UpperCamelCase					:					Optional[int]=None     ,		UpperCamelCase					:					str=False     ,		UpperCamelCase					:					Any=False     ,		**UpperCamelCase					:					Dict     ,		):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				AddedToken(UpperCamelCase     ,		lstrip=UpperCamelCase     ,		rstrip=UpperCamelCase				) if isinstance(UpperCamelCase     ,		UpperCamelCase				) else bos_token
               lowercase__       =				AddedToken(UpperCamelCase     ,		lstrip=UpperCamelCase     ,		rstrip=UpperCamelCase				) if isinstance(UpperCamelCase     ,		UpperCamelCase				) else eos_token
               lowercase__       =				AddedToken(UpperCamelCase     ,		lstrip=UpperCamelCase     ,		rstrip=UpperCamelCase				) if isinstance(UpperCamelCase     ,		UpperCamelCase				) else unk_token
               lowercase__       =				AddedToken(UpperCamelCase     ,		lstrip=UpperCamelCase     ,		rstrip=UpperCamelCase				) if isinstance(UpperCamelCase     ,		UpperCamelCase				) else pad_token
               super().__init__(
                   errors=UpperCamelCase     ,		unk_token=UpperCamelCase     ,		bos_token=UpperCamelCase     ,		eos_token=UpperCamelCase     ,		pad_token=UpperCamelCase     ,		add_prefix_space=UpperCamelCase     ,		add_bos_token=UpperCamelCase     ,		**UpperCamelCase     ,		)
               lowercase__       =				add_bos_token
  
               with open(UpperCamelCase     ,		encoding='''utf-8'''				) as vocab_handle:
                      lowercase__       =				json.load(UpperCamelCase				)
               lowercase__       =				{v: k for k, v in self.encoder.items()}
               lowercase__       =				errors  # how to handle errors in decoding
               lowercase__       =				bytes_to_unicode()
               lowercase__       =				{v: k for k, v in self.byte_encoder.items()}
               with open(UpperCamelCase     ,		encoding='''utf-8'''				) as merges_handle:
                      lowercase__       =				merges_handle.read().split('''\n'''				)[1:-1]
               lowercase__       =				[tuple(merge.split()				) for merge in bpe_merges]
               lowercase__       =				dict(zip(UpperCamelCase     ,		range(len(UpperCamelCase				)				)				)				)
               lowercase__       =				{}
               lowercase__       =				add_prefix_space
  
               # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
               lowercase__       =				re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''				)
  
  
  
        @property
        def    UpperCamelCase__	(self					:					Any				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               return len(self.encoder				)
  
  
  
        def    UpperCamelCase__	(self					:					Optional[Any]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               return dict(self.encoder     ,		**self.added_tokens_encoder				)
  
  
  
        def    UpperCamelCase__	(self					:					Union[str, Any]     ,		UpperCamelCase					:					Optional[int]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               if token in self.cache:
                      return self.cache[token]
               lowercase__       =				tuple(UpperCamelCase				)
               lowercase__       =				get_pairs(UpperCamelCase				)
  
               if not pairs:
                      return token
  
               while True:
                      lowercase__       =				min(UpperCamelCase     ,		key=lambda UpperCamelCase				: self.bpe_ranks.get(UpperCamelCase     ,		float('''inf'''				)				)				)
                      if bigram not in self.bpe_ranks:
                             break
                      lowercase__    ,lowercase__       =				bigram
                      lowercase__       =				[]
                      lowercase__       =				0
                      while i < len(UpperCamelCase				):
                             try:
                                    lowercase__       =				word.index(UpperCamelCase     ,		UpperCamelCase				)
                             except ValueError:
                                    new_word.extend(word[i:]				)
                                    break
                             else:
                                    new_word.extend(word[i:j]				)
                                    lowercase__       =				j
  
                             if word[i] == first and i < len(UpperCamelCase				) - 1 and word[i + 1] == second:
                                    new_word.append(first + second				)
                                    i += 2
                             else:
                                    new_word.append(word[i]				)
                                    i += 1
                      lowercase__       =				tuple(UpperCamelCase				)
                      lowercase__       =				new_word
                      if len(UpperCamelCase				) == 1:
                             break
                      else:
                             lowercase__       =				get_pairs(UpperCamelCase				)
               lowercase__       =				''' '''.join(UpperCamelCase				)
               lowercase__       =				word
               return word
  
  
  
        def    UpperCamelCase__	(self					:					Any     ,		UpperCamelCase					:					str     ,		UpperCamelCase					:					str=None				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               if self.add_bos_token:
                      lowercase__       =				[self.bos_token_id]
               else:
                      lowercase__       =				[]
  
               lowercase__       =				bos_token_ids + token_ids_a
  
               if token_ids_a is None:
                      return output
  
               return output + bos_token_ids + token_ids_a
  
  
  
        def    UpperCamelCase__	(self					:					Optional[Any]     ,		UpperCamelCase					:					str				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				[]
               for token in re.findall(self.pat     ,		UpperCamelCase				):
                      lowercase__       =				''''''.join(
                          self.byte_encoder[b] for b in token.encode('''utf-8'''				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                      bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase				).split(''' '''				)				)
               return bpe_tokens
  
  
  
        def    UpperCamelCase__	(self					:					str     ,		UpperCamelCase					:					str				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               return self.encoder.get(UpperCamelCase     ,		self.encoder.get(self.unk_token				)				)
  
  
  
        def    UpperCamelCase__	(self					:					Any     ,		UpperCamelCase					:					Optional[int]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               return self.decoder.get(UpperCamelCase				)
  
  
  
        def    UpperCamelCase__	(self					:					List[Any]     ,		UpperCamelCase					:					Optional[int]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				''''''.join(UpperCamelCase				)
               lowercase__       =				bytearray([self.byte_decoder[c] for c in text]				).decode('''utf-8'''     ,		errors=self.errors				)
               return text
  
  
  
        def    UpperCamelCase__	(self					:					int     ,		UpperCamelCase					:					str     ,		UpperCamelCase					:					Optional[str] = None				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               if not os.path.isdir(UpperCamelCase				):
                      logger.error(f"Vocabulary path ({save_directory}) should be a directory"				)
                      return
               lowercase__       =				os.path.join(
                   UpperCamelCase     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
               lowercase__       =				os.path.join(
                   UpperCamelCase     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)
  
               with open(UpperCamelCase     ,		'''w'''     ,		encoding='''utf-8'''				) as f:
                      f.write(json.dumps(self.encoder     ,		indent=2     ,		sort_keys=UpperCamelCase     ,		ensure_ascii=UpperCamelCase				) + '''\n'''				)
  
               lowercase__       =				0
               with open(UpperCamelCase     ,		'''w'''     ,		encoding='''utf-8'''				) as writer:
                      writer.write('''#version: 0.2\n'''				)
                      for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,		key=lambda UpperCamelCase				: kv[1]				):
                             if index != token_index:
                                    logger.warning(
                                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                                        ''' Please check that the tokenizer is not corrupted!'''				)
                                    lowercase__       =				token_index
                             writer.write(''' '''.join(UpperCamelCase				) + '''\n'''				)
                             index += 1
  
               return vocab_file, merge_file
  
  
  
        def    UpperCamelCase__	(self					:					str     ,		UpperCamelCase					:					List[Any]     ,		UpperCamelCase					:					int=False     ,		**UpperCamelCase					:					Optional[int]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				kwargs.pop('''add_prefix_space'''     ,		self.add_prefix_space				)
               if is_split_into_words or add_prefix_space:
                      lowercase__       =				''' ''' + text
               return (text, kwargs)
  
  
  
        def    UpperCamelCase__	(self					:					Tuple     ,		UpperCamelCase					:					Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]     ,		UpperCamelCase					:					bool = False     ,		UpperCamelCase					:					bool = None     ,		UpperCamelCase					:					Optional[List[str]] = None     ,		**UpperCamelCase					:					Optional[Any]     ,		):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				super()._decode(
                   token_ids=UpperCamelCase     ,		skip_special_tokens=UpperCamelCase     ,		clean_up_tokenization_spaces=UpperCamelCase     ,		**UpperCamelCase     ,		)
  
               if truncate_before_pattern is not None and len(UpperCamelCase				) > 0:
                      lowercase__       =				self.truncate(UpperCamelCase     ,		UpperCamelCase				)
  
               return decoded_text
  
  
  
  
  
  
        def    UpperCamelCase__	(self					:					str     ,		UpperCamelCase					:					List[str]     ,		UpperCamelCase					:					Union[str, Any]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               def find_re(UpperCamelCase					:					Dict     ,		UpperCamelCase					:					Dict     ,		UpperCamelCase					:					Optional[Any]				):
                      lowercase__       =				pattern.search(UpperCamelCase     ,		UpperCamelCase				)
                      return m.start() if m else -1
  
               lowercase__       =				[re.compile(UpperCamelCase     ,		re.MULTILINE				) for pattern in truncate_before_pattern]
  
               lowercase__       =				list(re.finditer('''^print'''     ,		UpperCamelCase     ,		re.MULTILINE				)				)
  
               if len(UpperCamelCase				) > 1:
                      lowercase__       =				completion[: prints[1].start()]
  
               lowercase__       =				list(re.finditer('''^def'''     ,		UpperCamelCase     ,		re.MULTILINE				)				)
  
               if len(UpperCamelCase				) > 1:
                      lowercase__       =				completion[: defs[1].start()]
  
               lowercase__       =				0
  
               lowercase__       =				[
                   pos for pos in [find_re(UpperCamelCase     ,		UpperCamelCase     ,		UpperCamelCase				) for terminal in terminals] if pos != -1
               ]
  
               if len(UpperCamelCase				) > 0:
                      return completion[: min(UpperCamelCase				)]
               else:
                      return completion
  
  
  
  
  