code: 
  
  
  '''simple docstring'''
  
  
  
  
  import os
  import re
  import unicodedata
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  
  import sentencepiece as spm
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  __lowercase  :      Optional[int]      		=				logging.get_logger(__name__)
  __lowercase  :      Optional[Any]      		=				{"""vocab_file""": """spiece.model"""}
  
  __lowercase  :      int      		=				{
      """vocab_file""": {
          """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/spiece.model""",
          """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/spiece.model""",
      },
  }
  
  __lowercase  :      Union[str, Any]      		=				{
      """google/fnet-base""": 5_12,
      """google/fnet-large""": 5_12,
  }
  
  __lowercase  :      Optional[Any]      		=				"""▁"""
  
  
  
  
  
  class _snake_case   ( a__   ):
  							"""simple docstring"""
  
  
  
  
  
  							__a												=     VOCAB_FILES_NAMES
  							__a												=     PRETRAINED_VOCAB_FILES_MAP
  							__a												=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  							__a												=     ['''input_ids''', '''token_type_ids''']
  
  
  
  
  
  
  
  							def __init__(   self			,				__A			,				__A=False			,				__A=True			,				__A=True			,				__A="<unk>"			,				__A="[SEP]"			,				__A="<pad>"			,				__A="[CLS]"			,				__A="[MASK]"			,				__A = None			,				**__A			,				)     ->None:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Dict											=					(
  											    AddedToken(__A			,				lstrip=__A			,				rstrip=__A			,				normalized=__A	)
  											    if isinstance(__A			,				__A	)
  											    else mask_token
  											)
  
  											__lowerCamelCase		:      List[Any]											=					{} if sp_model_kwargs is None else sp_model_kwargs
  
  											super().__init__(
  											    do_lower_case=__A			,				remove_space=__A			,				keep_accents=__A			,				unk_token=__A			,				sep_token=__A			,				pad_token=__A			,				cls_token=__A			,				mask_token=__A			,				sp_model_kwargs=self.sp_model_kwargs			,				**__A			,				)
  
  											__lowerCamelCase		:      Optional[int]											=					do_lower_case
  											__lowerCamelCase		:      str											=					remove_space
  											__lowerCamelCase		:      int											=					keep_accents
  											__lowerCamelCase		:      Tuple											=					vocab_file
  
  											__lowerCamelCase		:      Tuple											=					spm.SentencePieceProcessor(**self.sp_model_kwargs	)
  											self.sp_model.Load(__A	)
  
  
  
  
  
  
  
  							@property
  							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											return len(self.sp_model	)
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Dict											=					{self.convert_ids_to_tokens(__A	): i for i in range(self.vocab_size	)}
  											vocab.update(self.added_tokens_encoder	)
  											return vocab
  
  
  
  
  
  
  
  							def __getstate__(   self	)     ->Optional[Any]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      int											=					self.__dict__.copy()
  											__lowerCamelCase		:      int											=					None
  											return state
  
  
  
  
  
  
  
  							def __setstate__(   self			,				__A	)     ->Dict:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      int											=					d
  
  											# for backward compatibility
  											if not hasattr(self			,				"sp_model_kwargs"	):
  															__lowerCamelCase		:      int											=					{}
  
  											__lowerCamelCase		:      Tuple											=					spm.SentencePieceProcessor(**self.sp_model_kwargs	)
  											self.sp_model.Load(self.vocab_file	)
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->int:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											if self.remove_space:
  															__lowerCamelCase		:      Optional[Any]											=					" ".join(inputs.strip().split()	)
  											else:
  															__lowerCamelCase		:      Tuple											=					inputs
  											__lowerCamelCase		:      List[Any]											=					outputs.replace("``"			,				"\""	).replace("''"			,				"\""	)
  
  											if not self.keep_accents:
  															__lowerCamelCase		:      str											=					unicodedata.normalize("NFKD"			,				__A	)
  															__lowerCamelCase		:      str											=					"".join([c for c in outputs if not unicodedata.combining(__A	)]	)
  											if self.do_lower_case:
  															__lowerCamelCase		:      int											=					outputs.lower()
  
  											return outputs
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->List[str]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      List[Any]											=					self.preprocess_text(__A	)
  											__lowerCamelCase		:      int											=					self.sp_model.encode(__A			,				out_type=__A	)
  											__lowerCamelCase		:      Union[str, Any]											=					[]
  											for piece in pieces:
  															if len(__A	) > 1 and piece[-1] == str(","	) and piece[-2].isdigit():
  																			__lowerCamelCase		:      int											=					self.sp_model.EncodeAsPieces(piece[:-1].replace(__A			,				""	)	)
  																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
  																							if len(cur_pieces[0]	) == 1:
  																											__lowerCamelCase		:      List[str]											=					cur_pieces[1:]
  																							else:
  																											__lowerCamelCase		:      Any											=					cur_pieces[0][1:]
  																			cur_pieces.append(piece[-1]	)
  																			new_pieces.extend(__A	)
  															else:
  																			new_pieces.append(__A	)
  
  											return new_pieces
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->Dict:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											return self.sp_model.PieceToId(__A	)
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->Optional[Any]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											return self.sp_model.IdToPiece(__A	)
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->Optional[Any]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Any											=					[]
  											__lowerCamelCase		:      Union[str, Any]											=					""
  											__lowerCamelCase		:      int											=					False
  											for token in tokens:
  															# make sure that special tokens are not decoded using sentencepiece model
  															if token in self.all_special_tokens:
  																			if not prev_is_special:
  																							out_string += " "
  																			out_string += self.sp_model.decode(__A	) + token
  																			__lowerCamelCase		:      Tuple											=					True
  																			__lowerCamelCase		:      str											=					[]
  															else:
  																			current_sub_tokens.append(__A	)
  																			__lowerCamelCase		:      List[str]											=					False
  											out_string += self.sp_model.decode(__A	)
  											return out_string.strip()
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = False			,				__A = None			,				__A = True			,				**__A			,				)     ->str:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Dict											=					kwargs.pop("use_source_tokenizer"			,				__A	)
  
  											__lowerCamelCase		:      int											=					self.convert_ids_to_tokens(__A			,				skip_special_tokens=__A	)
  
  											# To avoid mixing byte-level and unicode for byte-level BPT
  											# we need to build string separately for added tokens and byte-level tokens
  											# cf. https://github.com/huggingface/transformers/issues/1133
  											__lowerCamelCase		:      int											=					[]
  											__lowerCamelCase		:      Optional[int]											=					[]
  											for token in filtered_tokens:
  															if skip_special_tokens and token in self.all_special_ids:
  																			continue
  															if token in self.added_tokens_encoder:
  																			if current_sub_text:
  																							sub_texts.append(self.convert_tokens_to_string(__A	)	)
  																							__lowerCamelCase		:      List[str]											=					[]
  																			sub_texts.append(__A	)
  															else:
  																			current_sub_text.append(__A	)
  											if current_sub_text:
  															sub_texts.append(self.convert_tokens_to_string(__A	)	)
  
  											# Mimic the behavior of the Rust tokenizer:
  											# No space after <unk>
  											if spaces_between_special_tokens:
  															__lowerCamelCase		:      Optional[int]											=					re.sub(r"(<unk>) "			,				r"\1"			,				" ".join(__A	)	)
  											else:
  															__lowerCamelCase		:      Optional[Any]											=					"".join(__A	)
  
  											__lowerCamelCase		:      int											=					(
  											    clean_up_tokenization_spaces
  											    if clean_up_tokenization_spaces is not None
  											    else self.clean_up_tokenization_spaces
  											)
  											if clean_up_tokenization_spaces:
  															__lowerCamelCase		:      Tuple											=					self.clean_up_tokenization(__A	)
  															return clean_text
  											else:
  															return text
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None	)     ->List[int]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      str											=					[self.sep_token_id]
  											__lowerCamelCase		:      int											=					[self.cls_token_id]
  											if token_ids_a is None:
  															return cls + token_ids_a + sep
  											return cls + token_ids_a + sep + token_ids_a + sep
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None			,				__A = False	)     ->List[int]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											if already_has_special_tokens:
  															return super().get_special_tokens_mask(
  															    token_ids_a=__A			,				token_ids_a=__A			,				already_has_special_tokens=__A	)
  
  											if token_ids_a is not None:
  															return [1] + ([0] * len(__A	)) + [1] + ([0] * len(__A	)) + [1]
  											return [1] + ([0] * len(__A	)) + [1]
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None	)     ->List[int]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Tuple											=					[self.sep_token_id]
  											__lowerCamelCase		:      Optional[int]											=					[self.cls_token_id]
  
  											if token_ids_a is None:
  															return len(cls + token_ids_a + sep	) * [0]
  											return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None	)     ->Tuple[str]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											if not os.path.isdir(__A	):
  															logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
  															return
  											__lowerCamelCase		:      Optional[Any]											=					os.path.join(
  											    __A			,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)
  
  											if os.path.abspath(self.vocab_file	) != os.path.abspath(__A	) and os.path.isfile(self.vocab_file	):
  															copyfile(self.vocab_file			,				__A	)
  											elif not os.path.isfile(self.vocab_file	):
  															with open(__A			,				"wb"	) as fi:
  																			__lowerCamelCase		:      Tuple											=					self.sp_model.serialized_model_proto()
  																			fi.write(__A	)
  
  											return (out_vocab_file,)
  
  
  
  
  
  
style_context: 
  
  
  '''simple docstring'''
  
  
  
  
  from __future__ import annotations
  
  from collections import deque
  from enum import Enum
  from math import atana, degrees
  from sys import maxsize
  
  
  
  
  
  class _snake_case   ( a__   ):
  							"""simple docstring"""
  
  
  
  
  
  							__a												=     1
  							__a												=     2
  							__a												=     3
  
  
  
  
  
  
  
  							def __repr__(   self	)     ->Tuple:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											return f'{self.__class__.__name__}.{self.name}'
  
  
  
  
  
  def 							a_       (						lowercase							,						lowercase							,						lowercase    )			->float:
  
  
  
  
  
  
  				'''simple docstring'''
  
  				__lowerCamelCase						,					__lowerCamelCase		:      str											=					point
  				return degrees(atana(y - miny							,						x - minx    )    )
  
  
  
  
  
  def 							a_       (						lowercase							,						lowercase							,						lowercase    )			->Direction:
  
  
  
  
  
  
  				'''simple docstring'''
  
  				__lowerCamelCase						,					__lowerCamelCase		:      List[str]											=					starting
  				__lowerCamelCase						,					__lowerCamelCase		:      Any											=					via
  				__lowerCamelCase						,					__lowerCamelCase		:      str											=					target
  				__lowerCamelCase		:      str											=					degrees(atana(ya - ya							,						xa - xa    )    )
  				via_angle %= 360
  				__lowerCamelCase		:      List[Any]											=					degrees(atana(ya - ya							,						xa - xa    )    )
  				target_angle %= 360
  				# t-
  				#  \ \
  				#   \ v
  				#    \|
  				#     s
  				# via_angle is always lower than target_angle, if direction is left.
  				# If they are same, it means they are on a same line of convex hull.
  				if target_angle > via_angle:
  								return Direction.left
  				elif target_angle == via_angle:
  								return Direction.straight
  				else:
  								return Direction.right
  
  
  
  
  
  def 							a_       (						lowercase    )			->list[tuple[int, int]]:
  
  
  
  
  
  
  				'''simple docstring'''
  
  				if len(lowercase    ) <= 2:
  								# There is no convex hull
  								raise ValueError("graham_scan: argument must contain more than 3 points."    )
  				if len(lowercase    ) == 3:
  								return points
  				# find the lowest and the most left point
  				__lowerCamelCase		:      Optional[Any]											=					0
  				__lowerCamelCase						,					__lowerCamelCase		:      List[Any]											=					maxsize, maxsize
  				for i, point in enumerate(lowercase    ):
  								__lowerCamelCase		:      Dict											=					point[0]
  								__lowerCamelCase		:      Optional[int]											=					point[1]
  								if y < miny:
  												__lowerCamelCase		:      Dict											=					y
  												__lowerCamelCase		:      int											=					x
  												__lowerCamelCase		:      List[Any]											=					i
  								if y == miny and x < minx:
  												__lowerCamelCase		:      Optional[int]											=					x
  												__lowerCamelCase		:      Any											=					i
  
      # remove the lowest and the most left point from points for preparing for sort
  				points.pop(lowercase    )
  
  				__lowerCamelCase		:      Optional[Any]											=					sorted(lowercase							,						key=lambda lowercase    : angle_comparer(lowercase							,						lowercase							,						lowercase    )    )
  				# This insert actually costs complexity,
  				# and you should instead add (minx, miny) into stack later.
  				# I'm using insert just for easy understanding.
  				sorted_points.insert(0							,						(minx, miny)    )
  
  				__lowerCamelCase		:      deque[tuple[int, int]]											=					deque()
  				stack.append(sorted_points[0]    )
  				stack.append(sorted_points[1]    )
  				stack.append(sorted_points[2]    )
  				# In any ways, the first 3 points line are towards left.
  				# Because we sort them the angle from minx, miny.
  				__lowerCamelCase		:      str											=					Direction.left
  
  				for i in range(3							,						len(lowercase    )    ):
  								while True:
  												__lowerCamelCase		:      List[Any]											=					stack[-2]
  												__lowerCamelCase		:      str											=					stack[-1]
  												__lowerCamelCase		:      int											=					sorted_points[i]
  												__lowerCamelCase		:      List[Any]											=					check_direction(lowercase							,						lowercase							,						lowercase    )
  
  												if next_direction == Direction.left:
  																__lowerCamelCase		:      Optional[int]											=					Direction.left
  																break
  												if next_direction == Direction.straight:
  																if current_direction == Direction.left:
  																				# We keep current_direction as left.
  																				# Because if the straight line keeps as straight,
  																				# we want to know if this straight line is towards left.
  																				break
  																elif current_direction == Direction.right:
  																				# If the straight line is towards right,
  																				# every previous points on those straigh line is not convex hull.
  																				stack.pop()
  												if next_direction == Direction.right:
  																stack.pop()
  								stack.append(sorted_points[i]    )
  				return list(lowercase    )
  
  
  
  
  
  