



'''simple docstring'''






import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCamelCase__:   Optional[int]								=			logging.get_logger(__name__)

UpperCamelCase__:   Union[str, Any]								=			{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__:   int								=			{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

UpperCamelCase__:   int								=			{
    "Salesforce/codegen-350M-mono": 2048,
}

class 				SCREAMING_SNAKE_CASE(							UpperCAmelCase__			):







	"""simple docstring"""






	lowerCamelCase__			 =       VOCAB_FILES_NAMES
	lowerCamelCase__			 =       PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase__			 =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCamelCase__			 =       ["input_ids", "attention_mask"]
	lowerCamelCase__			 =       CodeGenTokenizer







	def __init__(						self			:     Any		,			__snake_case			:     str=None		,			__snake_case			:     Union[str, Any]=None		,			__snake_case			:     List[str]=None		,			__snake_case			:     Dict="<|endoftext|>"		,			__snake_case			:     Optional[int]="<|endoftext|>"		,			__snake_case			:     Tuple="<|endoftext|>"		,			__snake_case			:     int=False		,			**__snake_case			:     Tuple		,			)			->		str:
								super().__init__(
								    lowerCamelCase__		,			lowerCamelCase__		,			tokenizer_file=lowerCamelCase__		,			unk_token=lowerCamelCase__		,			bos_token=lowerCamelCase__		,			eos_token=lowerCamelCase__		,			add_prefix_space=lowerCamelCase__		,			**lowerCamelCase__		,			)

								if kwargs.pop('''add_bos_token'''		,			lowerCamelCase__		):
															UpperCAmelCase :       Optional[Any] 							=		kwargs.pop('''name_or_path'''		,			''''''		)
															raise ValueError(
															    '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
															    '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
															    F"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
															    F"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
															    '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
															    ''' so that the fast tokenizer works correctly.'''		)

								UpperCAmelCase :       Union[str, Any] 							=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
								if pre_tok_state.get('''add_prefix_space'''		,			lowerCamelCase__		) != add_prefix_space:
															UpperCAmelCase :       Optional[Any] 							=		getattr(lowerCamelCase__		,			pre_tok_state.pop('''type'''		)		)
															UpperCAmelCase :       int 							=		add_prefix_space
															UpperCAmelCase :       int 							=		pre_tok_class(**lowerCamelCase__		)

								UpperCAmelCase :       List[Any] 							=		add_prefix_space







	def      A	(						self			:     Optional[int]		,			*__snake_case			:     int		,			**__snake_case			:     Any		)			->		BatchEncoding:
								UpperCAmelCase :       Tuple 							=		kwargs.get('''is_split_into_words'''		,			lowerCamelCase__		)
								assert self.add_prefix_space or not is_split_into_words, (
								    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*lowerCamelCase__		,			**lowerCamelCase__		)







	def      A	(						self			:     Optional[int]		,			*__snake_case			:     int		,			**__snake_case			:     Any		)			->		BatchEncoding:
								UpperCAmelCase :       Any 							=		kwargs.get('''is_split_into_words'''		,			lowerCamelCase__		)

								assert self.add_prefix_space or not is_split_into_words, (
								    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*lowerCamelCase__		,			**lowerCamelCase__		)







	def      A	(						self			:     Optional[int]		,			__snake_case			:     str		,			__snake_case			:     Optional[str] = None		)			->		Tuple[str]:
								UpperCAmelCase :       Optional[int] 							=		self._tokenizer.model.save(lowerCamelCase__		,			name=lowerCamelCase__		)
								return tuple(lowerCamelCase__		)







	def      A	(						self			:     Optional[int]		,			__snake_case			:     Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]		,			__snake_case			:     bool = False		,			__snake_case			:     bool = None		,			__snake_case			:     Optional[List[str]] = None		,			**__snake_case			:     Union[str, Any]		,			)			->		str:
								UpperCAmelCase :       str 							=		super().decode(
								    token_ids=lowerCamelCase__		,			skip_special_tokens=lowerCamelCase__		,			clean_up_tokenization_spaces=lowerCamelCase__		,			**lowerCamelCase__		,			)

								if truncate_before_pattern is not None and len(lowerCamelCase__		) > 0:
															UpperCAmelCase :       int 							=		self.truncate(lowerCamelCase__		,			lowerCamelCase__		)

								return decoded_text







	def      A	(						self			:     int		,			__snake_case			:     int		,			__snake_case			:     int		)			->		Tuple:







								def find_re(__snake_case			:     Optional[Any]		,			__snake_case			:     Optional[Any]		,			__snake_case			:     Tuple		):
															UpperCAmelCase :       Optional[int] 							=		pattern.search(lowerCamelCase__		,			lowerCamelCase__		)
															return m.start() if m else -1

								UpperCAmelCase :       Dict 							=		[re.compile(lowerCamelCase__		,			re.MULTILINE		) for pattern in truncate_before_pattern]

								UpperCAmelCase :       Any 							=		list(re.finditer('''^print'''		,			lowerCamelCase__		,			re.MULTILINE		)		)

								if len(lowerCamelCase__		) > 1:
															UpperCAmelCase :       Tuple 							=		completion[: prints[1].start()]

								UpperCAmelCase :       Dict 							=		list(re.finditer('''^def'''		,			lowerCamelCase__		,			re.MULTILINE		)		)

								if len(lowerCamelCase__		) > 1:
															UpperCAmelCase :       Optional[Any] 							=		completion[: defs[1].start()]

								UpperCAmelCase :       List[str] 							=		0

								UpperCAmelCase :       Tuple 							=		[
								    pos for pos in [find_re(lowerCamelCase__		,			lowerCamelCase__		,			lowerCamelCase__		) for terminal in terminals] if pos != -1
								]

								if len(lowerCamelCase__		) > 0:
															return completion[: min(lowerCamelCase__		)]
								else:
															return completion









'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_fnet import FNetTokenizer
else:
						lowerCamelCase__					=   None

lowerCamelCase__					=   logging.get_logger(__name__)
lowerCamelCase__					=   {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase__					=   {
    'vocab_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/spiece.model',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json',
    },
}

lowerCamelCase__					=   {
    'google/fnet-base': 512,
    'google/fnet-large': 512,
}

lowerCamelCase__					=   '▁'






class 		lowerCAmelCase__	(	UpperCAmelCase__		):
			lowerCAmelCase       :		Dict				      =   VOCAB_FILES_NAMES
			lowerCAmelCase       :		Optional[Any]				      =   PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase       :		List[str]				      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase       :		Optional[int]				      =   ["input_ids", "token_type_ids"]
			lowerCAmelCase       :		Optional[Any]				      =   FNetTokenizer




			def __init__(						self     :   Dict		,     lowerCamelCase__     :   int=None		,     lowerCamelCase__     :   List[Any]=None		,     lowerCamelCase__     :   Any=False		,     lowerCamelCase__     :   List[str]=True		,     lowerCamelCase__     :   Dict=True		,     lowerCamelCase__     :   str="<unk>"		,     lowerCamelCase__     :   List[str]="[SEP]"		,     lowerCamelCase__     :   Union[str, Any]="<pad>"		,     lowerCamelCase__     :   Optional[Any]="[CLS]"		,     lowerCamelCase__     :   Any="[MASK]"		,     **lowerCamelCase__     :   Any		,     )      ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				Dict	    =					(
					    AddedToken(lowerCamelCase__		,     lstrip=lowerCamelCase__		,     rstrip=lowerCamelCase__		,     normalized=lowerCamelCase__       )
					    if isinstance(lowerCamelCase__		,     lowerCamelCase__       )
					    else mask_token
					)

					super().__init__(
					    lowerCamelCase__		,     tokenizer_file=lowerCamelCase__		,     do_lower_case=lowerCamelCase__		,     remove_space=lowerCamelCase__		,     keep_accents=lowerCamelCase__		,     unk_token=lowerCamelCase__		,     sep_token=lowerCamelCase__		,     pad_token=lowerCamelCase__		,     cls_token=lowerCamelCase__		,     mask_token=lowerCamelCase__		,     **lowerCamelCase__		,     )

					_UpperCAmelCase						:				Optional[Any]	    =					do_lower_case
					_UpperCAmelCase						:				Tuple	    =					remove_space
					_UpperCAmelCase						:				List[Any]	    =					keep_accents
					_UpperCAmelCase						:				Tuple	    =					vocab_file
					_UpperCAmelCase						:				str	    =					False if not self.vocab_file else True




			def 							lowerCAmelCase__			(						self     :   str		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None       )      ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[Any]	    =					[self.sep_token_id]
					_UpperCAmelCase						:				Tuple	    =					[self.cls_token_id]
					if token_ids_a is None:
							return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep




			def 							lowerCAmelCase__			(						self     :   int		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None       )      ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[int]	    =					[self.sep_token_id]
					_UpperCAmelCase						:				Optional[Any]	    =					[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






			def 							lowerCAmelCase__			(						self     :   Dict		,     lowerCamelCase__     :   str		,     lowerCamelCase__     :   Optional[str] = None       )      ->Tuple[str]:



					'''simple docstring'''







					if not os.path.isdir(lowerCamelCase__       ):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
							return
					_UpperCAmelCase						:				Union[str, Any]	    =					os.path.join(
					    lowerCamelCase__		,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

					if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowerCamelCase__       ):
							copyfile(self.vocab_file		,     lowerCamelCase__       )

					return (out_vocab_file,)



