




import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_   							=      logging.get_logger(__name__)
UpperCAmelCase_   							=      {'vocab_file': 'spiece.model'}

UpperCAmelCase_   							=      {
    'vocab_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/spiece.model',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/spiece.model',
    },
}

UpperCAmelCase_   							=      {
    'google/fnet-base': 512,
    'google/fnet-large': 512,
}

UpperCAmelCase_   							=      '▁'





class        lowerCamelCase__(   __UpperCamelCase):
      UpperCAmelCase__ :      Union[str, Any]       						=  VOCAB_FILES_NAMES
      UpperCAmelCase__ :      int       						=  PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase__ :      Tuple       						=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase__ :      List[str]       						=  ["input_ids", "token_type_ids"]
      def __init__(							self:					List[str]       ,			UpperCamelCase_:					Optional[int]       ,			UpperCamelCase_:					Union[str, Any]=False       ,			UpperCamelCase_:					Optional[int]=True       ,			UpperCamelCase_:					int=True       ,			UpperCamelCase_:					Dict="<unk>"       ,			UpperCamelCase_:					List[str]="[SEP]"       ,			UpperCamelCase_:					Tuple="<pad>"       ,			UpperCamelCase_:					Union[str, Any]="[CLS]"       ,			UpperCamelCase_:					Any="[MASK]"       ,			UpperCamelCase_:					Optional[Dict[str, Any]] = None       ,			**UpperCamelCase_:					Union[str, Any]       ,			):
             # Mask token behave like a normal word, i.e. include the space before it and
             # is included in the raw text, there should be a match in a non-normalized sentence.
             __lowerCamelCase  =		(
                 AddedToken(_lowerCAmelCase       ,			lstrip=_lowerCAmelCase       ,			rstrip=_lowerCAmelCase       ,			normalized=_lowerCAmelCase  )
                 if isinstance(_lowerCAmelCase       ,			_lowerCAmelCase  )
                 else mask_token
             )

             __lowerCamelCase  =		{} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=_lowerCAmelCase       ,			remove_space=_lowerCAmelCase       ,			keep_accents=_lowerCAmelCase       ,			unk_token=_lowerCAmelCase       ,			sep_token=_lowerCAmelCase       ,			pad_token=_lowerCAmelCase       ,			cls_token=_lowerCAmelCase       ,			mask_token=_lowerCAmelCase       ,			sp_model_kwargs=self.sp_model_kwargs       ,			**_lowerCAmelCase       ,			)

             __lowerCamelCase  =		do_lower_case
             __lowerCamelCase  =		remove_space
             __lowerCamelCase  =		keep_accents
             __lowerCamelCase  =		vocab_file

             __lowerCamelCase  =		spm.SentencePieceProcessor(**self.sp_model_kwargs  )
             self.sp_model.Load(_lowerCAmelCase  )
      @property
      def      lowerCAmelCase__		(							self:					Dict  ):
             return len(self.sp_model  )
      def      lowerCAmelCase__		(							self:					str  ):
             __lowerCamelCase  =		{self.convert_ids_to_tokens(_lowerCAmelCase  ): i for i in range(self.vocab_size  )}
             vocab.update(self.added_tokens_encoder  )
             return vocab
      def __getstate__(							self:					Optional[Any]  ):
             __lowerCamelCase  =		self.__dict__.copy()
             __lowerCamelCase  =		None
             return state
      def __setstate__(							self:					List[Any]       ,			UpperCamelCase_:					Union[str, Any]  ):
             __lowerCamelCase  =		d

             # for backward compatibility
             if not hasattr(self       ,			"""sp_model_kwargs"""  ):
                    __lowerCamelCase  =		{}

             __lowerCamelCase  =		spm.SentencePieceProcessor(**self.sp_model_kwargs  )
             self.sp_model.Load(self.vocab_file  )
      def      lowerCAmelCase__		(							self:					Any       ,			UpperCamelCase_:					Any  ):
             if self.remove_space:
                    __lowerCamelCase  =		""" """.join(inputs.strip().split()  )
             else:
                    __lowerCamelCase  =		inputs
             __lowerCamelCase  =		outputs.replace("""``"""       ,			"""\""""  ).replace("""''"""       ,			"""\""""  )

             if not self.keep_accents:
                    __lowerCamelCase  =		unicodedata.normalize("""NFKD"""       ,			_lowerCAmelCase  )
                    __lowerCamelCase  =		"""""".join([c for c in outputs if not unicodedata.combining(_lowerCAmelCase  )]  )
             if self.do_lower_case:
                    __lowerCamelCase  =		outputs.lower()

             return outputs
      def      lowerCAmelCase__		(							self:					Union[str, Any]       ,			UpperCamelCase_:					str  ):
             __lowerCamelCase  =		self.preprocess_text(_lowerCAmelCase  )
             __lowerCamelCase  =		self.sp_model.encode(_lowerCAmelCase       ,			out_type=_lowerCAmelCase  )
             __lowerCamelCase  =		[]
             for piece in pieces:
                    if len(_lowerCAmelCase  ) > 1 and piece[-1] == str(""","""  ) and piece[-2].isdigit():
                           __lowerCamelCase  =		self.sp_model.EncodeAsPieces(piece[:-1].replace(_lowerCAmelCase       ,			""""""  )  )
                           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                  if len(cur_pieces[0]  ) == 1:
                                         __lowerCamelCase  =		cur_pieces[1:]
                                  else:
                                         __lowerCamelCase  =		cur_pieces[0][1:]
                           cur_pieces.append(piece[-1]  )
                           new_pieces.extend(_lowerCAmelCase  )
                    else:
                           new_pieces.append(_lowerCAmelCase  )

             return new_pieces
      def      lowerCAmelCase__		(							self:					str       ,			UpperCamelCase_:					List[Any]  ):
             return self.sp_model.PieceToId(_lowerCAmelCase  )
      def      lowerCAmelCase__		(							self:					List[str]       ,			UpperCamelCase_:					int  ):
             return self.sp_model.IdToPiece(_lowerCAmelCase  )
      def      lowerCAmelCase__		(							self:					Optional[int]       ,			UpperCamelCase_:					Union[str, Any]  ):
             __lowerCamelCase  =		[]
             __lowerCamelCase  =		""""""
             __lowerCamelCase  =		False
             for token in tokens:
                    # make sure that special tokens are not decoded using sentencepiece model
                    if token in self.all_special_tokens:
                           if not prev_is_special:
                                  out_string += " "
                           out_string += self.sp_model.decode(_lowerCAmelCase  ) + token
                           __lowerCamelCase  =		True
                           __lowerCamelCase  =		[]
                    else:
                           current_sub_tokens.append(_lowerCAmelCase  )
                           __lowerCamelCase  =		False
             out_string += self.sp_model.decode(_lowerCAmelCase  )
             return out_string.strip()
      def      lowerCAmelCase__		(							self:					str       ,			UpperCamelCase_:					List[int]       ,			UpperCamelCase_:					bool = False       ,			UpperCamelCase_:					bool = None       ,			UpperCamelCase_:					bool = True       ,			**UpperCamelCase_:					str       ,			):
             __lowerCamelCase  =		kwargs.pop("""use_source_tokenizer"""       ,			_lowerCAmelCase  )

             __lowerCamelCase  =		self.convert_ids_to_tokens(_lowerCAmelCase       ,			skip_special_tokens=_lowerCAmelCase  )

             # To avoid mixing byte-level and unicode for byte-level BPT
             # we need to build string separately for added tokens and byte-level tokens
             # cf. https://github.com/huggingface/transformers/issues/1133
             __lowerCamelCase  =		[]
             __lowerCamelCase  =		[]
             for token in filtered_tokens:
                    if skip_special_tokens and token in self.all_special_ids:
                           continue
                    if token in self.added_tokens_encoder:
                           if current_sub_text:
                                  sub_texts.append(self.convert_tokens_to_string(_lowerCAmelCase  )  )
                                  __lowerCamelCase  =		[]
                           sub_texts.append(_lowerCAmelCase  )
                    else:
                           current_sub_text.append(_lowerCAmelCase  )
             if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(_lowerCAmelCase  )  )

             # Mimic the behavior of the Rust tokenizer:
             # No space after <unk>
             if spaces_between_special_tokens:
                    __lowerCamelCase  =		re.sub(r"""(<unk>) """       ,			r"""\1"""       ,			""" """.join(_lowerCAmelCase  )  )
             else:
                    __lowerCamelCase  =		"""""".join(_lowerCAmelCase  )

             __lowerCamelCase  =		(
                 clean_up_tokenization_spaces
                 if clean_up_tokenization_spaces is not None
                 else self.clean_up_tokenization_spaces
             )
             if clean_up_tokenization_spaces:
                    __lowerCamelCase  =		self.clean_up_tokenization(_lowerCAmelCase  )
                    return clean_text
             else:
                    return text
      def      lowerCAmelCase__		(							self:					Dict       ,			UpperCamelCase_:					List[int]       ,			UpperCamelCase_:					Optional[List[int]] = None  ):
             __lowerCamelCase  =		[self.sep_token_id]
             __lowerCamelCase  =		[self.cls_token_id]
             if token_ids_a is None:
                    return cls + token_ids_a + sep
             return cls + token_ids_a + sep + token_ids_a + sep
      def      lowerCAmelCase__		(							self:					Optional[Any]       ,			UpperCamelCase_:					List[int]       ,			UpperCamelCase_:					Optional[List[int]] = None       ,			UpperCamelCase_:					bool = False  ):

             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=_lowerCAmelCase       ,			token_ids_a=_lowerCAmelCase       ,			already_has_special_tokens=_lowerCAmelCase  )

             if token_ids_a is not None:
                    return [1] + ([0] * len(_lowerCAmelCase  )) + [1] + ([0] * len(_lowerCAmelCase  )) + [1]
             return [1] + ([0] * len(_lowerCAmelCase  )) + [1]
      def      lowerCAmelCase__		(							self:					Union[str, Any]       ,			UpperCamelCase_:					List[int]       ,			UpperCamelCase_:					Optional[List[int]] = None  ):
             __lowerCamelCase  =		[self.sep_token_id]
             __lowerCamelCase  =		[self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep  ) * [0]
             return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




      def      lowerCAmelCase__		(							self:					Union[str, Any]       ,			UpperCamelCase_:					str       ,			UpperCamelCase_:					Optional[str] = None  ):
             if not os.path.isdir(_lowerCAmelCase  ):
                    logger.error(F'Vocabulary path ({save_directory}) should be a directory'  )
                    return
             __lowerCamelCase  =		os.path.join(
                 _lowerCAmelCase       ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

             if os.path.abspath(self.vocab_file  ) != os.path.abspath(_lowerCAmelCase  ) and os.path.isfile(self.vocab_file  ):
                    copyfile(self.vocab_file       ,			_lowerCAmelCase  )
             elif not os.path.isfile(self.vocab_file  ):
                    with open(_lowerCAmelCase       ,			"""wb"""  ) as fi:
                           __lowerCamelCase  =		self.sp_model.serialized_model_proto()
                           fi.write(_lowerCAmelCase  )

             return (out_vocab_file,)


from __future__ import annotations

from collections.abc import Iterable







class      SCREAMING_SNAKE_CASE__					:

       def __init__(							self					:     Optional[Any]			,			_lowerCAmelCase					:     Tuple			,			_lowerCAmelCase					:     Tuple			):
           __snake_case			, __snake_case       :				str 			=    float(_lowerCAmelCase			), float(_lowerCAmelCase			)

       def __eq__(							self					:     List[Any]			,			_lowerCAmelCase					:     List[str]			):
           return self.x == other.x and self.y == other.y

       def __ne__(							self					:     Any			,			_lowerCAmelCase					:     Optional[Any]			):
           return not self == other

       def __gt__(							self					:     Union[str, Any]			,			_lowerCAmelCase					:     int			):
           if self.x > other.x:
               return True
           elif self.x == other.x:
               return self.y > other.y
           return False

       def __lt__(							self					:     List[str]			,			_lowerCAmelCase					:     List[str]			):
           return not self > other

       def __ge__(							self					:     List[Any]			,			_lowerCAmelCase					:     List[Any]			):
           if self.x > other.x:
               return True
           elif self.x == other.x:
               return self.y >= other.y
           return False

       def __le__(							self					:     List[Any]			,			_lowerCAmelCase					:     List[Any]			):
           if self.x < other.x:
               return True
           elif self.x == other.x:
               return self.y <= other.y
           return False

       def __repr__(							self					:     str			):
           return f'''({self.x}, {self.y})'''





       def __hash__(							self					:     List[str]			):
           return hash(self.x			)







def 					__lowerCAmelCase				(		__SCREAMING_SNAKE_CASE       :					list[Point] | list[list[float]] | Iterable[list[float]]		,							):







    '''simple docstring'''
    __snake_case       :				list[Point] 			=    []
    if list_of_tuples:
        for p in list_of_tuples:
            if isinstance(__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE   ):
                points.append(__SCREAMING_SNAKE_CASE   )
            else:
                try:
                    points.append(Point(p[0]		,							p[1]   )   )
                except (IndexError, TypeError):
                    print(
                        F'''Ignoring deformed point {p}. All points'''
                        """ must have at least 2 coordinates."""   )
    return points







def 					__lowerCAmelCase				(		__SCREAMING_SNAKE_CASE       :					list[Point] | list[list[float]]   ):







    '''simple docstring'''

    if not hasattr(__SCREAMING_SNAKE_CASE		,							"""__iter__"""   ):
        __snake_case       :				List[str] 			=    F'''Expecting an iterable object but got an non-iterable type {points}'''
        raise ValueError(__SCREAMING_SNAKE_CASE   )

    if not points:
        __snake_case       :				Tuple 			=    F'''Expecting a list of points but got {points}'''
        raise ValueError(__SCREAMING_SNAKE_CASE   )

    return _construct_points(__SCREAMING_SNAKE_CASE   )







def 					__lowerCAmelCase				(		__SCREAMING_SNAKE_CASE       :					Point		,							__SCREAMING_SNAKE_CASE       :					Point		,							__SCREAMING_SNAKE_CASE       :					Point   ):







    '''simple docstring'''
    __snake_case       :				List[Any] 			=    (a.x * b.y + b.x * c.y + c.x * a.y) - (a.y * b.x + b.y * c.x + c.y * a.x)
    return det







def 					__lowerCAmelCase				(		__SCREAMING_SNAKE_CASE       :					list[Point]   ):







    '''simple docstring'''
    __snake_case       :				List[Any] 			=    sorted(_validate_input(__SCREAMING_SNAKE_CASE   )   )
    __snake_case       :				Optional[Any] 			=    len(__SCREAMING_SNAKE_CASE   )
    __snake_case       :				Dict 			=    set()

    for i in range(n - 1   ):
        for j in range(i + 1		,							__SCREAMING_SNAKE_CASE   ):
            __snake_case       :				str 			=    False
            __snake_case       :				Any 			=    True
            for k in range(__SCREAMING_SNAKE_CASE   ):
                if k != i and k != j:
                    __snake_case       :				str 			=    _det(points[i]		,							points[j]		,							points[k]   )

                    if det_k > 0:
                        __snake_case       :				str 			=    True
                    elif det_k < 0:
                        __snake_case       :				List[Any] 			=    True
                    else:
                        # point[i], point[j], point[k] all lie on a straight line
                        # if point[k] is to the left of point[i] or it's to the
                        # right of point[j], then point[i], point[j] cannot be
                        # part of the convex hull of A
                        if points[k] < points[i] or points[k] > points[j]:
                            __snake_case       :				Any 			=    False
                            break

                if points_left_of_ij and points_right_of_ij:
                    __snake_case       :				int 			=    False
                    break

            if ij_part_of_convex_hull:
                convex_set.update([points[i], points[j]]   )

    return sorted(__SCREAMING_SNAKE_CASE   )







def 					__lowerCAmelCase				(		__SCREAMING_SNAKE_CASE       :					list[Point]   ):







    '''simple docstring'''
    __snake_case       :				Union[str, Any] 			=    sorted(_validate_input(__SCREAMING_SNAKE_CASE   )   )
    __snake_case       :				Any 			=    len(__SCREAMING_SNAKE_CASE   )

    # divide all the points into an upper hull and a lower hull
    # the left most point and the right most point are definitely
    # members of the convex hull by definition.
    # use these two anchors to divide all the points into two hulls,
    # an upper hull and a lower hull.

    # all points to the left (above) the line joining the extreme points belong to the
    # upper hull
    # all points to the right (below) the line joining the extreme points below to the
    # lower hull
    # ignore all points on the line joining the extreme points since they cannot be
    # part of the convex hull

    __snake_case       :				Tuple 			=    points[0]
    __snake_case       :				Optional[Any] 			=    points[n - 1]

    __snake_case       :				Optional[Any] 			=    {left_most_point, right_most_point}
    __snake_case       :				Optional[int] 			=    []
    __snake_case       :				List[str] 			=    []

    for i in range(1		,							n - 1   ):
        __snake_case       :				Tuple 			=    _det(__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							points[i]   )

        if det > 0:
            upper_hull.append(points[i]   )
        elif det < 0:
            lower_hull.append(points[i]   )

    _construct_hull(__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE   )
    _construct_hull(__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE   )

    return sorted(__SCREAMING_SNAKE_CASE   )







def 					__lowerCAmelCase				(		__SCREAMING_SNAKE_CASE       :					list[Point]		,							__SCREAMING_SNAKE_CASE       :					Point		,							__SCREAMING_SNAKE_CASE       :					Point		,							__SCREAMING_SNAKE_CASE       :					set[Point]   ):







    '''simple docstring'''
    if points:
        __snake_case       :				List[str] 			=    None
        __snake_case       :				Tuple 			=    float("""-inf"""   )
        __snake_case       :				List[Any] 			=    []

        for p in points:
            __snake_case       :				Union[str, Any] 			=    _det(__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE   )

            if det > 0:
                candidate_points.append(__SCREAMING_SNAKE_CASE   )

                if det > extreme_point_distance:
                    __snake_case       :				List[str] 			=    det
                    __snake_case       :				List[str] 			=    p

        if extreme_point:
            _construct_hull(__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE   )
            convex_set.add(__SCREAMING_SNAKE_CASE   )
            _construct_hull(__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE   )







def 					__lowerCAmelCase				(		__SCREAMING_SNAKE_CASE       :					list[Point]   ):







    '''simple docstring'''
    __snake_case       :				Tuple 			=    sorted(_validate_input(__SCREAMING_SNAKE_CASE   )   )
    __snake_case       :				str 			=    len(__SCREAMING_SNAKE_CASE   )

    __snake_case       :				Optional[int] 			=    points[:2]
    for i in range(2		,							__SCREAMING_SNAKE_CASE   ):
        __snake_case       :				List[str] 			=    _det(convex_hull[1]		,							convex_hull[0]		,							points[i]   )
        if det > 0:
            convex_hull.insert(0		,							points[i]   )
            break
        elif det < 0:
            convex_hull.append(points[i]   )
            break
        else:
            __snake_case       :				Union[str, Any] 			=    points[i]
    i += 1

    for j in range(__SCREAMING_SNAKE_CASE		,							__SCREAMING_SNAKE_CASE   ):
        if (
            _det(convex_hull[0]		,							convex_hull[-1]		,							points[j]   ) > 0
            and _det(convex_hull[-1]		,							convex_hull[0]		,							points[1]   ) < 0
        ):
            # The point lies within the convex hull
            continue

        convex_hull.insert(0		,							points[j]   )
        convex_hull.append(points[j]   )
        while _det(convex_hull[0]		,							convex_hull[1]		,							convex_hull[2]   ) >= 0:
            del convex_hull[1]
        while _det(convex_hull[-1]		,							convex_hull[-2]		,							convex_hull[-3]   ) <= 0:
            del convex_hull[-2]

    # `convex_hull` is contains the convex hull in circular order
    return sorted(convex_hull[1:] if len(__SCREAMING_SNAKE_CASE   ) > 3 else convex_hull   )







def 					__lowerCAmelCase				(		):







    '''simple docstring'''
    __snake_case       :				str 			=    [
        (0, 3),
        (2, 2),
        (1, 1),
        (2, 1),
        (3, 0),
        (0, 0),
        (3, 3),
        (2, -1),
        (2, -4),
        (1, -3),
    ]
    # the convex set of points is
    # [(0, 0), (0, 3), (1, -3), (2, -4), (3, 0), (3, 3)]
    __snake_case       :				Optional[Any] 			=    convex_hull_bf(__SCREAMING_SNAKE_CASE   )

    __snake_case       :				List[Any] 			=    convex_hull_recursive(__SCREAMING_SNAKE_CASE   )
    assert results_bf == results_recursive

    __snake_case       :				Tuple 			=    convex_hull_melkman(__SCREAMING_SNAKE_CASE   )
    assert results_bf == results_melkman

    print(__SCREAMING_SNAKE_CASE   )


if __name__ == "__main__":
     main()



