





'''simple docstring'''







import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
					if is_torch_available():
										import torch
					if is_tf_available():
										import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


lowerCAmelCase_					:    List[str]      	=			logging.get_logger(__name__)

lowerCAmelCase_					:    str      	=			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase_					:    Any      	=			{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

lowerCAmelCase_					:    str      	=			{
    'Salesforce/codegen-350M-mono': 20_48,
}
class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






				"""simple docstring"""







				__a            =VOCAB_FILES_NAMES
				__a            =PRETRAINED_VOCAB_FILES_MAP
				__a            =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a            =['input_ids', 'attention_mask']
				__a            =CodeGenTokenizer




				def __init__(  self     :					Optional[Any]					,				__a     :					Optional[int]=None					,				__a     :					Optional[Any]=None					,				__a     :					Union[str, Any]=None					,				__a     :					List[str]="<|endoftext|>"					,				__a     :					Union[str, Any]="<|endoftext|>"					,				__a     :					int="<|endoftext|>"					,				__a     :					Dict=False					,				**__a     :					Any					,				):
								super().__init__(
								    __a					,				__a					,				tokenizer_file=__a					,				unk_token=__a					,				bos_token=__a					,				eos_token=__a					,				add_prefix_space=__a					,				**__a					,				)

								if kwargs.pop("add_bos_token"					,				__a					):
												_a								= kwargs.pop("name_or_path"					,				""					)
												raise ValueError(
												    "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
												    "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
												    f'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
												    f'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
												    "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
												    " so that the fast tokenizer works correctly."					)

								_a								= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
								if pre_tok_state.get("add_prefix_space"					,				__a					) != add_prefix_space:
												_a								= getattr(__a					,				pre_tok_state.pop("type"					)					)
												_a								= add_prefix_space
												_a								= pre_tok_class(**__a					)

								_a								= add_prefix_space




				def   UpperCamelCase__   (  self     :					Any					,				*__a     :					int					,				**__a     :					Any					):
								_a								= kwargs.get("is_split_into_words"					,				__a					)
								assert self.add_prefix_space or not is_split_into_words, (
								    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*__a					,				**__a					)




				def   UpperCamelCase__   (  self     :					str					,				*__a     :					Optional[Any]					,				**__a     :					Union[str, Any]					):
								_a								= kwargs.get("is_split_into_words"					,				__a					)

								assert self.add_prefix_space or not is_split_into_words, (
								    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*__a					,				**__a					)




				def   UpperCamelCase__   (  self     :					Dict					,				__a     :					str					,				__a     :					Optional[str] = None					):
								_a								= self._tokenizer.model.save(__a					,				name=__a					)
								return tuple(__a					)




				def   UpperCamelCase__   (  self     :					Optional[Any]					,				__a     :					Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]					,				__a     :					bool = False					,				__a     :					bool = None					,				__a     :					Optional[List[str]] = None					,				**__a     :					Union[str, Any]					,				):
								_a								= super().decode(
								    token_ids=__a					,				skip_special_tokens=__a					,				clean_up_tokenization_spaces=__a					,				**__a					,				)

								if truncate_before_pattern is not None and len(__a					) > 0:
												_a								= self.truncate(__a					,				__a					)

								return decoded_text




				def   UpperCamelCase__   (  self     :					List[str]					,				__a     :					int					,				__a     :					Dict					):
								def find_re(__a     :					Optional[int]					,				__a     :					Any					,				__a     :					List[str]					):
												_a								= pattern.search(__a					,				__a					)
												return m.start() if m else -1

								_a								= [re.compile(__a					,				re.MULTILINE					) for pattern in truncate_before_pattern]

								_a								= list(re.finditer("^print"					,				__a					,				re.MULTILINE					)					)

								if len(__a					) > 1:
												_a								= completion[: prints[1].start()]

								_a								= list(re.finditer("^def"					,				__a					,				re.MULTILINE					)					)

								if len(__a					) > 1:
												_a								= completion[: defs[1].start()]

								_a								= 0

								_a								= [
								    pos for pos in [find_re(__a					,				__a					,				__a					) for terminal in terminals] if pos != -1
								]

								if len(__a					) > 0:
												return completion[: min(__a					)]
								else:
												return completion








import warnings


warnings.warn(
    'memory_utils has been reorganized to utils.memory. Import `find_executable_batchsize` from the main `__init__`: '
    '`from accelerate import find_executable_batch_size` to avoid this warning.',
    FutureWarning,
)




