code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  import colorsys
  import io
  
  import cva
  import matplotlib as mpl
  import matplotlib.colors as mplc
  import matplotlib.figure as mplfigure
  import numpy as np
  import torch
  from matplotlib.backends.backend_agg import FigureCanvasAgg
  
  from utils import img_tensorize
  
  
  lowercase__			=	1_0_0_0
  
  
  
  
  
  
  class 						lowercase     :
  
  
  
  
  
  
        def __init__(self ,  a__ ,  a__=1.2 ,  a__="g" ,  a__=0.5 ,  a__="-" ,  a__="test_out.jpg" ,  a__=True ,  a__=False ,  a__=None ,  a__=None ,  a__=0.7 ,  )       ->   int:
           if isinstance(a__ ,  torch.Tensor					):
              SCREAMING_SNAKE_CASE__			:				str       						= img.numpy().astype('''np.uint8'''					)
           if isinstance(a__ ,  a__					):
              SCREAMING_SNAKE_CASE__			:				str       						= img_tensorize(a__					)
           assert isinstance(a__ ,  np.ndarray					)
  
           SCREAMING_SNAKE_CASE__						,				SCREAMING_SNAKE_CASE__			:				Dict       						= img.shape[1], img.shape[0]
           SCREAMING_SNAKE_CASE__			:				List[str]       						= mplfigure.Figure(frameon=a__					)
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= fig.get_dpi()
           SCREAMING_SNAKE_CASE__			:				Tuple       						= (width * scale + 1E-2) / dpi
           SCREAMING_SNAKE_CASE__			:				Dict       						= (height * scale + 1E-2) / dpi
           fig.set_size_inches(a__ ,  a__					)
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= fig.add_axes([0.0, 0.0, 1.0, 1.0]					)
           ax.axis('''off'''					)
           ax.set_xlim(0.0 ,  a__					)
           ax.set_ylim(a__					)
  
           SCREAMING_SNAKE_CASE__			:				int       						= saveas
           SCREAMING_SNAKE_CASE__			:				Tuple       						= rgb
           SCREAMING_SNAKE_CASE__			:				List[str]       						= pynb
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= img
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= edgecolor
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= 0.5
           SCREAMING_SNAKE_CASE__			:				Any       						= linestyle
           SCREAMING_SNAKE_CASE__			:				Any       						= int(np.sqrt(min(a__ ,  a__					)					) * scale // 3					)
           SCREAMING_SNAKE_CASE__			:				Dict       						= width
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= height
           SCREAMING_SNAKE_CASE__			:				Any       						= scale
           SCREAMING_SNAKE_CASE__			:				List[str]       						= fig
           SCREAMING_SNAKE_CASE__			:				Any       						= ax
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= pad
           SCREAMING_SNAKE_CASE__			:				str       						= idaobj
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= idaattr
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= FigureCanvasAgg(a__					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__=None					)       ->   Optional[int]:
           if color is None:
              SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= self.edgecolor
           ((SCREAMING_SNAKE_CASE__)						,				(SCREAMING_SNAKE_CASE__)						,				(SCREAMING_SNAKE_CASE__)						,				(SCREAMING_SNAKE_CASE__))			:				Optional[int]       						= box
           SCREAMING_SNAKE_CASE__			:				List[str]       						= xa - xa
           SCREAMING_SNAKE_CASE__			:				Tuple       						= ya - ya
           self.ax.add_patch(
               mpl.patches.Rectangle(
                   (xa, ya) ,  a__ ,  a__ ,  fill=a__ ,  edgecolor=a__ ,  linewidth=self.font_size // 3 ,  alpha=self.alpha ,  linestyle=self.linestyle ,  )					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__=None ,  a__=None ,  a__=None ,  a__=None					)       ->   Dict:
           if len(boxes.shape					) > 2:
              SCREAMING_SNAKE_CASE__			:				Optional[int]       						= boxes[0]
           if len(obj_ids.shape					) > 1:
              SCREAMING_SNAKE_CASE__			:				Optional[int]       						= obj_ids[0]
           if len(obj_scores.shape					) > 1:
              SCREAMING_SNAKE_CASE__			:				Any       						= obj_scores[0]
           if len(attr_ids.shape					) > 1:
              SCREAMING_SNAKE_CASE__			:				str       						= attr_ids[0]
           if len(attr_scores.shape					) > 1:
              SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= attr_scores[0]
           if isinstance(a__ ,  torch.Tensor					):
              SCREAMING_SNAKE_CASE__			:				List[Any]       						= boxes.numpy()
           if isinstance(a__ ,  a__					):
              SCREAMING_SNAKE_CASE__			:				int       						= np.array(a__					)
           assert isinstance(a__ ,  np.ndarray					)
           SCREAMING_SNAKE_CASE__			:				Any       						= np.prod(boxes[:, 2:] - boxes[:, :2] ,  axis=1					)
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= np.argsort(-areas					).tolist()
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= boxes[sorted_idxs] if boxes is not None else None
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= obj_ids[sorted_idxs] if obj_ids is not None else None
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= obj_scores[sorted_idxs] if obj_scores is not None else None
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= attr_ids[sorted_idxs] if attr_ids is not None else None
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= attr_scores[sorted_idxs] if attr_scores is not None else None
  
           SCREAMING_SNAKE_CASE__			:				str       						= [self._random_color(maximum=1					) for _ in range(len(a__					)					)]
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [assigned_colors[idx] for idx in sorted_idxs]
           if obj_ids is not None:
              SCREAMING_SNAKE_CASE__			:				List[Any]       						= self._create_text_labels_attr(a__ ,  a__ ,  a__ ,  a__					)
              for i in range(len(a__					)					):
                 SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= assigned_colors[i]
                 self.add_box(boxes[i] ,  a__					)
                 self.draw_labels(labels[i] ,  boxes[i] ,  a__					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__					)       ->   str:
           SCREAMING_SNAKE_CASE__						,				SCREAMING_SNAKE_CASE__						,				SCREAMING_SNAKE_CASE__						,				SCREAMING_SNAKE_CASE__			:				str       						= box
           SCREAMING_SNAKE_CASE__			:				Any       						= (xa, ya)
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= (ya - ya) * (xa - xa)
           SCREAMING_SNAKE_CASE__			:				Tuple       						= _SMALL_OBJ * self.scale
           if instance_area < small or ya - ya < 4_0 * self.scale:
              if ya >= self.height - 5:
                 SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= (xa, ya)
              else:
                 SCREAMING_SNAKE_CASE__			:				Optional[int]       						= (xa, ya)
  
           SCREAMING_SNAKE_CASE__			:				Any       						= (ya - ya) / np.sqrt(self.height * self.width					)
           SCREAMING_SNAKE_CASE__			:				Tuple       						= self._change_color_brightness(a__ ,  brightness_factor=0.7					)
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= np.clip((height_ratio - 0.02) / 0.08 + 1 ,  1.2 ,  2					)
           font_size *= 0.75 * self.font_size
  
           self.draw_text(
               text=a__ ,  position=a__ ,  color=a__ ,  )
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__="g" ,  a__="left" ,  )       ->   Optional[int]:
           SCREAMING_SNAKE_CASE__			:				str       						= 0
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= self.font_size
           SCREAMING_SNAKE_CASE__			:				Dict       						= np.maximum(list(mplc.to_rgb(a__					)					) ,  0.2					)
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= max(0.8 ,  np.max(a__					)					)
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= {
               '''facecolor''': '''black''',
               '''alpha''': self.alpha,
               '''pad''': self.pad,
               '''edgecolor''': '''none''',
           }
           SCREAMING_SNAKE_CASE__						,				SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= position
           self.ax.text(
               a__ ,  a__ ,  a__ ,  size=font_size * self.scale ,  family='''sans-serif''' ,  bbox=a__ ,  verticalalignment='''top''' ,  horizontalalignment=a__ ,  color=a__ ,  zorder=1_0 ,  rotation=a__ ,  )
  
  
  
  
  
  
        def    A__(self ,  a__=None					)       ->   Union[str, Any]:
           if saveas is None:
              SCREAMING_SNAKE_CASE__			:				List[Any]       						= self.saveas
           if saveas.lower().endswith('''.jpg'''					) or saveas.lower().endswith('''.png'''					):
              cva.imwrite(
                  a__ ,  self._get_buffer()[:, :, ::-1] ,  )
           else:
              self.fig.savefig(a__					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Optional[int]:
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= [self.idaobj[i] for i in classes]
           SCREAMING_SNAKE_CASE__			:				str       						= [self.idaattr[i] for i in attr_classes]
           SCREAMING_SNAKE_CASE__			:				Any       						= [
               f"""{label} {score:.2f} {attr} {attr_score:.2f}"""
               for label, score, attr, attr_score in zip(a__ ,  a__ ,  a__ ,  a__					)
           ]
           return labels
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__					)       ->   Optional[int]:
           SCREAMING_SNAKE_CASE__			:				Dict       						= [self.idaobj[i] for i in classes]
           if scores is not None:
              if labels is None:
                 SCREAMING_SNAKE_CASE__			:				int       						= ['''{:.0f}%'''.format(s * 1_0_0					) for s in scores]
              else:
                 SCREAMING_SNAKE_CASE__			:				str       						= ['''{} {:.0f}%'''.format(a__ ,  s * 1_0_0					) for li, s in zip(a__ ,  a__					)]
           return labels
  
  
  
  
  
  
        def    A__(self ,  a__=2_5_5					)       ->   str:
           SCREAMING_SNAKE_CASE__			:				str       						= np.random.randint(0 ,  len(_COLORS					)					)
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= _COLORS[idx] * maximum
           if not self.rgb:
              SCREAMING_SNAKE_CASE__			:				Tuple       						= ret[::-1]
           return ret
  
  
  
  
  
  
        def    A__(self					)       ->   Union[str, Any]:
           if not self.pynb:
              SCREAMING_SNAKE_CASE__						,				SCREAMING_SNAKE_CASE__			:				Optional[int]       						= self.canvas.print_to_buffer()
              if (width, height) != (self.width, self.height):
                 SCREAMING_SNAKE_CASE__			:				List[str]       						= cva.resize(self.img ,  (width, height)					)
              else:
                 SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= self.img
           else:
              SCREAMING_SNAKE_CASE__			:				Optional[int]       						= io.BytesIO()  # works for cairo backend
              self.canvas.print_rgba(a__					)
              SCREAMING_SNAKE_CASE__						,				SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= self.width, self.height
              SCREAMING_SNAKE_CASE__			:				Dict       						= buf.getvalue()
              SCREAMING_SNAKE_CASE__			:				Optional[int]       						= self.img
  
           SCREAMING_SNAKE_CASE__			:				str       						= np.frombuffer(a__ ,  dtype='''uint8'''					)
           SCREAMING_SNAKE_CASE__			:				int       						= buffer.reshape(a__ ,  a__ ,  4					)
           SCREAMING_SNAKE_CASE__						,				SCREAMING_SNAKE_CASE__			:				Any       						= np.split(a__ ,  [3] ,  axis=2					)
  
           try:
              import numexpr as ne  # fuse them with numexpr
  
              SCREAMING_SNAKE_CASE__			:				Dict       						= ne.evaluate('''img * (1 - alpha / 255.0) + rgb * (alpha / 255.0)'''					)
           except ImportError:
              SCREAMING_SNAKE_CASE__			:				int       						= alpha.astype('''float32'''					) / 2_55.0
              SCREAMING_SNAKE_CASE__			:				Optional[int]       						= img * (1 - alpha) + rgb * alpha
  
           return visualized_image.astype('''uint8'''					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__					)       ->   Tuple:
           assert brightness_factor >= -1.0 and brightness_factor <= 1.0
           SCREAMING_SNAKE_CASE__			:				List[str]       						= mplc.to_rgb(a__					)
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= colorsys.rgb_to_hls(*mplc.to_rgb(a__					)					)
           SCREAMING_SNAKE_CASE__			:				List[str]       						= polygon_color[1] + (brightness_factor * polygon_color[1])
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= 0.0 if modified_lightness < 0.0 else modified_lightness
           SCREAMING_SNAKE_CASE__			:				Dict       						= 1.0 if modified_lightness > 1.0 else modified_lightness
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= colorsys.hls_to_rgb(polygon_color[0] ,  a__ ,  polygon_color[2]					)
           return modified_color
  
  lowercase__			=	(
      np.array(
          [
              0.000,
              0.447,
              0.741,
              0.850,
              0.325,
              0.098,
              0.929,
              0.694,
              0.125,
              0.494,
              0.184,
              0.556,
              0.466,
              0.674,
              0.188,
              0.301,
              0.745,
              0.933,
              0.635,
              0.078,
              0.184,
              0.300,
              0.300,
              0.300,
              0.600,
              0.600,
              0.600,
              1.000,
              0.000,
              0.000,
              1.000,
              0.500,
              0.000,
              0.749,
              0.749,
              0.000,
              0.000,
              1.000,
              0.000,
              0.000,
              0.000,
              1.000,
              0.667,
              0.000,
              1.000,
              0.333,
              0.333,
              0.000,
              0.333,
              0.667,
              0.000,
              0.333,
              1.000,
              0.000,
              0.667,
              0.333,
              0.000,
              0.667,
              0.667,
              0.000,
              0.667,
              1.000,
              0.000,
              1.000,
              0.333,
              0.000,
              1.000,
              0.667,
              0.000,
              1.000,
              1.000,
              0.000,
              0.000,
              0.333,
              0.500,
              0.000,
              0.667,
              0.500,
              0.000,
              1.000,
              0.500,
              0.333,
              0.000,
              0.500,
              0.333,
              0.333,
              0.500,
              0.333,
              0.667,
              0.500,
              0.333,
              1.000,
              0.500,
              0.667,
              0.000,
              0.500,
              0.667,
              0.333,
              0.500,
              0.667,
              0.667,
              0.500,
              0.667,
              1.000,
              0.500,
              1.000,
              0.000,
              0.500,
              1.000,
              0.333,
              0.500,
              1.000,
              0.667,
              0.500,
              1.000,
              1.000,
              0.500,
              0.000,
              0.333,
              1.000,
              0.000,
              0.667,
              1.000,
              0.000,
              1.000,
              1.000,
              0.333,
              0.000,
              1.000,
              0.333,
              0.333,
              1.000,
              0.333,
              0.667,
              1.000,
              0.333,
              1.000,
              1.000,
              0.667,
              0.000,
              1.000,
              0.667,
              0.333,
              1.000,
              0.667,
              0.667,
              1.000,
              0.667,
              1.000,
              1.000,
              1.000,
              0.000,
              1.000,
              1.000,
              0.333,
              1.000,
              1.000,
              0.667,
              1.000,
              0.333,
              0.000,
              0.000,
              0.500,
              0.000,
              0.000,
              0.667,
              0.000,
              0.000,
              0.833,
              0.000,
              0.000,
              1.000,
              0.000,
              0.000,
              0.000,
              0.167,
              0.000,
              0.000,
              0.333,
              0.000,
              0.000,
              0.500,
              0.000,
              0.000,
              0.667,
              0.000,
              0.000,
              0.833,
              0.000,
              0.000,
              1.000,
              0.000,
              0.000,
              0.000,
              0.167,
              0.000,
              0.000,
              0.333,
              0.000,
              0.000,
              0.500,
              0.000,
              0.000,
              0.667,
              0.000,
              0.000,
              0.833,
              0.000,
              0.000,
              1.000,
              0.000,
              0.000,
              0.000,
              0.143,
              0.143,
              0.143,
              0.857,
              0.857,
              0.857,
              1.000,
              1.000,
              1.000,
          ]
      )
      .astype(np.floataa)
      .reshape(-1, 3)
  )
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  import os
  import re
  import unicodedata
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  
  import sentencepiece as spm
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  lowercase__			=	logging.get_logger(__name__)
  lowercase__			=	{'''vocab_file''': '''spiece.model'''}
  
  lowercase__			=	{
      '''vocab_file''': {
          '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
          '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
      },
  }
  
  lowercase__			=	{
      '''google/fnet-base''': 5_1_2,
      '''google/fnet-large''': 5_1_2,
  }
  
  lowercase__			=	'''▁'''
  
  
  
  
  
  
  class 						lowercase     (   UpperCAmelCase_):
        __lowercase 		=  VOCAB_FILES_NAMES
        __lowercase 		=  PRETRAINED_VOCAB_FILES_MAP
        __lowercase 		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        __lowercase 		=  ['''input_ids''', '''token_type_ids''']
  
  
  
  
  
  
        def __init__(self ,  a__ ,  a__=False ,  a__=True ,  a__=True ,  a__="<unk>" ,  a__="[SEP]" ,  a__="<pad>" ,  a__="[CLS]" ,  a__="[MASK]" ,  a__ = None ,  **a__ ,  )       ->   None:
           # Mask token behave like a normal word, i.e. include the space before it and
           # is included in the raw text, there should be a match in a non-normalized sentence.
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= (
               AddedToken(a__ ,  lstrip=a__ ,  rstrip=a__ ,  normalized=a__					)
               if isinstance(a__ ,  a__					)
               else mask_token
           )
  
           SCREAMING_SNAKE_CASE__			:				Any       						= {} if sp_model_kwargs is None else sp_model_kwargs
  
           super().__init__(
               do_lower_case=a__ ,  remove_space=a__ ,  keep_accents=a__ ,  unk_token=a__ ,  sep_token=a__ ,  pad_token=a__ ,  cls_token=a__ ,  mask_token=a__ ,  sp_model_kwargs=self.sp_model_kwargs ,  **a__ ,  )
  
           SCREAMING_SNAKE_CASE__			:				Dict       						= do_lower_case
           SCREAMING_SNAKE_CASE__			:				Tuple       						= remove_space
           SCREAMING_SNAKE_CASE__			:				List[str]       						= keep_accents
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= vocab_file
  
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
           self.sp_model.Load(a__					)
  
  
  
  
  
  
        @property
        def    A__(self					)       ->   Union[str, Any]:
           return len(self.sp_model					)
  
  
  
  
  
  
        def    A__(self					)       ->   Any:
           SCREAMING_SNAKE_CASE__			:				int       						= {self.convert_ids_to_tokens(a__					): i for i in range(self.vocab_size					)}
           vocab.update(self.added_tokens_encoder					)
           return vocab
  
  
  
  
  
  
        def __getstate__(self					)       ->   Any:
           SCREAMING_SNAKE_CASE__			:				List[str]       						= self.__dict__.copy()
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= None
           return state
  
  
  
  
  
  
        def __setstate__(self ,  a__					)       ->   int:
           SCREAMING_SNAKE_CASE__			:				Any       						= d
  
           # for backward compatibility
           if not hasattr(self ,  '''sp_model_kwargs'''					):
              SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= {}
  
           SCREAMING_SNAKE_CASE__			:				int       						= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
           self.sp_model.Load(self.vocab_file					)
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   str:
           if self.remove_space:
              SCREAMING_SNAKE_CASE__			:				List[str]       						= ''' '''.join(inputs.strip().split()					)
           else:
              SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= inputs
           SCREAMING_SNAKE_CASE__			:				Dict       						= outputs.replace('''``''' ,  '''"'''					).replace('''\'\'''' ,  '''"'''					)
  
           if not self.keep_accents:
              SCREAMING_SNAKE_CASE__			:				Dict       						= unicodedata.normalize('''NFKD''' ,  a__					)
              SCREAMING_SNAKE_CASE__			:				List[Any]       						= ''''''.join([c for c in outputs if not unicodedata.combining(a__					)]					)
           if self.do_lower_case:
              SCREAMING_SNAKE_CASE__			:				str       						= outputs.lower()
  
           return outputs
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   List[str]:
           SCREAMING_SNAKE_CASE__			:				Dict       						= self.preprocess_text(a__					)
           SCREAMING_SNAKE_CASE__			:				List[str]       						= self.sp_model.encode(a__ ,  out_type=a__					)
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= []
           for piece in pieces:
              if len(a__					) > 1 and piece[-1] == str(''','''					) and piece[-2].isdigit():
                 SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= self.sp_model.EncodeAsPieces(piece[:-1].replace(a__ ,  ''''''					)					)
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                    if len(cur_pieces[0]					) == 1:
                       SCREAMING_SNAKE_CASE__			:				List[str]       						= cur_pieces[1:]
                    else:
                       SCREAMING_SNAKE_CASE__			:				str       						= cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]					)
                 new_pieces.extend(a__					)
              else:
                 new_pieces.append(a__					)
  
           return new_pieces
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   str:
           return self.sp_model.PieceToId(a__					)
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   Optional[int]:
           return self.sp_model.IdToPiece(a__					)
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   Optional[Any]:
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= []
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= ''''''
           SCREAMING_SNAKE_CASE__			:				Any       						= False
           for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                 if not prev_is_special:
                    out_string += " "
                 out_string += self.sp_model.decode(a__					) + token
                 SCREAMING_SNAKE_CASE__			:				List[Any]       						= True
                 SCREAMING_SNAKE_CASE__			:				int       						= []
              else:
                 current_sub_tokens.append(a__					)
                 SCREAMING_SNAKE_CASE__			:				Dict       						= False
           out_string += self.sp_model.decode(a__					)
           return out_string.strip()
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = False ,  a__ = None ,  a__ = True ,  **a__ ,  )       ->   str:
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= kwargs.pop('''use_source_tokenizer''' ,  a__					)
  
           SCREAMING_SNAKE_CASE__			:				int       						= self.convert_ids_to_tokens(a__ ,  skip_special_tokens=a__					)
  
           # To avoid mixing byte-level and unicode for byte-level BPT
           # we need to build string separately for added tokens and byte-level tokens
           # cf. https://github.com/huggingface/transformers/issues/1133
           SCREAMING_SNAKE_CASE__			:				Dict       						= []
           SCREAMING_SNAKE_CASE__			:				List[str]       						= []
           for token in filtered_tokens:
              if skip_special_tokens and token in self.all_special_ids:
                 continue
              if token in self.added_tokens_encoder:
                 if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(a__					)					)
                    SCREAMING_SNAKE_CASE__			:				List[str]       						= []
                 sub_texts.append(a__					)
              else:
                 current_sub_text.append(a__					)
           if current_sub_text:
              sub_texts.append(self.convert_tokens_to_string(a__					)					)
  
           # Mimic the behavior of the Rust tokenizer:
           # No space after <unk>
           if spaces_between_special_tokens:
              SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= re.sub(R'''(<unk>) ''' ,  R'''\1''' ,  ''' '''.join(a__					)					)
           else:
              SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= ''''''.join(a__					)
  
           SCREAMING_SNAKE_CASE__			:				int       						= (
               clean_up_tokenization_spaces
               if clean_up_tokenization_spaces is not None
               else self.clean_up_tokenization_spaces
           )
           if clean_up_tokenization_spaces:
              SCREAMING_SNAKE_CASE__			:				str       						= self.clean_up_tokenization(a__					)
              return clean_text
           else:
              return text
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
           SCREAMING_SNAKE_CASE__			:				str       						= [self.sep_token_id]
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= [self.cls_token_id]
           if token_ids_a is None:
              return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None ,  a__ = False					)       ->   List[int]:
  
           if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=a__ ,  token_ids_a=a__ ,  already_has_special_tokens=a__					)
  
           if token_ids_a is not None:
              return [1] + ([0] * len(a__					)) + [1] + ([0] * len(a__					)) + [1]
           return [1] + ([0] * len(a__					)) + [1]
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
           SCREAMING_SNAKE_CASE__			:				List[str]       						= [self.sep_token_id]
           SCREAMING_SNAKE_CASE__			:				Tuple       						= [self.cls_token_id]
  
           if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
           return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   Tuple[str]:
           if not os.path.isdir(a__					):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
              return
           SCREAMING_SNAKE_CASE__			:				str       						= os.path.join(
               a__ ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
  
           if os.path.abspath(self.vocab_file					) != os.path.abspath(a__					) and os.path.isfile(self.vocab_file					):
              copyfile(self.vocab_file ,  a__					)
           elif not os.path.isfile(self.vocab_file					):
              with open(a__ ,  '''wb'''					) as fi:
                 SCREAMING_SNAKE_CASE__			:				int       						= self.sp_model.serialized_model_proto()
                 fi.write(a__					)
  
           return (out_vocab_file,)
  
  