
import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
				if is_torch_available():
								import torch
				if is_tf_available():
								import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__lowerCamelCase								=					logging.get_logger(__name__)

__lowerCamelCase								=					{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__lowerCamelCase								=					{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

__lowerCamelCase								=					{
    'Salesforce/codegen-350M-mono': 2_0_4_8,
}




@lru_cache()
def 		lowerCAmelCase_     (     )   ->      List[str]:

							"""simple docstring"""






							_UpperCAmelCase  :			Any	       =		(
							    list(range(ord("!"  )      ,	ord("~"  ) + 1  )  ) + list(range(ord("¡"  )      ,	ord("¬"  ) + 1  )  ) + list(range(ord("®"  )      ,	ord("ÿ"  ) + 1  )  )
							)
							_UpperCAmelCase  :			List[str]	       =		bs[:]
							_UpperCAmelCase  :			Any	       =		0
							for b in range(2**8  ):
														if b not in bs:
																					bs.append(_SCREAMING_SNAKE_CASE  )
																					cs.append(2**8 + n  )
																					n += 1
							_UpperCAmelCase  :			List[Any]	       =		[chr(_SCREAMING_SNAKE_CASE  ) for n in cs]
							return dict(zip(_SCREAMING_SNAKE_CASE      ,	_SCREAMING_SNAKE_CASE  )  )




def 		lowerCAmelCase_     (     _SCREAMING_SNAKE_CASE  )   ->      Union[str, Any]:

							"""simple docstring"""






							_UpperCAmelCase  :			List[Any]	       =		set()
							_UpperCAmelCase  :			Tuple	       =		word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)  )
														_UpperCAmelCase  :			Union[str, Any]	       =		char
							return pairs


class    _UpperCamelCase(   SCREAMING_SNAKE_CASE				):
						__A:						Optional[Any]				   =      VOCAB_FILES_NAMES
						__A:						Dict				   =      PRETRAINED_VOCAB_FILES_MAP
						__A:						Union[str, Any]				   =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__A:						Optional[Any]				   =      ["""input_ids""", """attention_mask"""]

						def __init__(					self		:			List[str]	,						_lowerCamelCase		:			Any	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			Dict="replace"	,						_lowerCamelCase		:			str="<|endoftext|>"	,						_lowerCamelCase		:			Union[str, Any]="<|endoftext|>"	,						_lowerCamelCase		:			Optional[Any]="<|endoftext|>"	,						_lowerCamelCase		:			Dict=None	,						_lowerCamelCase		:			Any=False	,						_lowerCamelCase		:			str=False	,						**_lowerCamelCase		:			Optional[int]	,						):
													_UpperCAmelCase  :			Dict	       =		AddedToken(_lowerCamelCase	,						lstrip=_lowerCamelCase	,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase	,						_lowerCamelCase			) else bos_token
													_UpperCAmelCase  :			List[Any]	       =		AddedToken(_lowerCamelCase	,						lstrip=_lowerCamelCase	,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase	,						_lowerCamelCase			) else eos_token
													_UpperCAmelCase  :			List[str]	       =		AddedToken(_lowerCamelCase	,						lstrip=_lowerCamelCase	,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase	,						_lowerCamelCase			) else unk_token
													_UpperCAmelCase  :			str	       =		AddedToken(_lowerCamelCase	,						lstrip=_lowerCamelCase	,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase	,						_lowerCamelCase			) else pad_token
													super().__init__(
													    errors=_lowerCamelCase	,						unk_token=_lowerCamelCase	,						bos_token=_lowerCamelCase	,						eos_token=_lowerCamelCase	,						pad_token=_lowerCamelCase	,						add_prefix_space=_lowerCamelCase	,						add_bos_token=_lowerCamelCase	,						**_lowerCamelCase	,						)
													_UpperCAmelCase  :			List[str]	       =		add_bos_token

													with open(_lowerCamelCase	,						encoding="utf-8"			) as vocab_handle:
																				_UpperCAmelCase  :			Optional[int]	       =		json.load(_lowerCamelCase			)
													_UpperCAmelCase  :			Optional[Any]	       =		{v: k for k, v in self.encoder.items()}
													_UpperCAmelCase  :			Optional[int]	       =		errors  # how to handle errors in decoding
													_UpperCAmelCase  :			List[str]	       =		bytes_to_unicode()
													_UpperCAmelCase  :			List[str]	       =		{v: k for k, v in self.byte_encoder.items()}
													with open(_lowerCamelCase	,						encoding="utf-8"			) as merges_handle:
																				_UpperCAmelCase  :			List[str]	       =		merges_handle.read().split("\n"			)[1:-1]
													_UpperCAmelCase  :			List[Any]	       =		[tuple(merge.split()			) for merge in bpe_merges]
													_UpperCAmelCase  :			Optional[Any]	       =		dict(zip(_lowerCamelCase	,						range(len(_lowerCamelCase			)			)			)			)
													_UpperCAmelCase  :			List[str]	       =		{}
													_UpperCAmelCase  :			List[str]	       =		add_prefix_space

													# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
													_UpperCAmelCase  :			Tuple	       =		re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"			)

						@property
						def 						a__ (					self		:			str			):
													return len(self.encoder			)

						def 						a__ (					self		:			Dict			):
													return dict(self.encoder	,						**self.added_tokens_encoder			)

						def 						a__ (					self		:			List[str]	,						_lowerCamelCase		:			Any			):
													if token in self.cache:
																				return self.cache[token]
													_UpperCAmelCase  :			Optional[int]	       =		tuple(_lowerCamelCase			)
													_UpperCAmelCase  :			Any	       =		get_pairs(_lowerCamelCase			)

													if not pairs:
																				return token

													while True:
																				_UpperCAmelCase  :			Any	       =		min(_lowerCamelCase	,						key=lambda _lowerCamelCase			: self.bpe_ranks.get(_lowerCamelCase	,						float("inf"			)			)			)
																				if bigram not in self.bpe_ranks:
																											break
																				_UpperCAmelCase       ,_UpperCAmelCase  :			List[str]	       =		bigram
																				_UpperCAmelCase  :			Dict	       =		[]
																				_UpperCAmelCase  :			str	       =		0
																				while i < len(_lowerCamelCase			):
																											try:
																																		_UpperCAmelCase  :			str	       =		word.index(_lowerCamelCase	,						_lowerCamelCase			)
																											except ValueError:
																																		new_word.extend(word[i:]			)
																																		break
																											else:
																																		new_word.extend(word[i:j]			)
																																		_UpperCAmelCase  :			Dict	       =		j

																											if word[i] == first and i < len(_lowerCamelCase			) - 1 and word[i + 1] == second:
																																		new_word.append(first + second			)
																																		i += 2
																											else:
																																		new_word.append(word[i]			)
																																		i += 1
																				_UpperCAmelCase  :			List[str]	       =		tuple(_lowerCamelCase			)
																				_UpperCAmelCase  :			str	       =		new_word
																				if len(_lowerCamelCase			) == 1:
																											break
																				else:
																											_UpperCAmelCase  :			str	       =		get_pairs(_lowerCamelCase			)
													_UpperCAmelCase  :			Optional[Any]	       =		" ".join(_lowerCamelCase			)
													_UpperCAmelCase  :			Dict	       =		word
													return word

						def 						a__ (					self		:			Dict	,						_lowerCamelCase		:			List[Any]	,						_lowerCamelCase		:			Dict=None			):
													if self.add_bos_token:
																				_UpperCAmelCase  :			Optional[int]	       =		[self.bos_token_id]
													else:
																				_UpperCAmelCase  :			Tuple	       =		[]

													_UpperCAmelCase  :			Tuple	       =		bos_token_ids + token_ids_a

													if token_ids_a is None:
																				return output

													return output + bos_token_ids + token_ids_a

						def 						a__ (					self		:			Optional[int]	,						_lowerCamelCase		:			Dict			):
													_UpperCAmelCase  :			int	       =		[]
													for token in re.findall(self.pat	,						_lowerCamelCase			):
																				_UpperCAmelCase  :			List[Any]	       =		"".join(
																				    self.byte_encoder[b] for b in token.encode("utf-8"			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																				bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase			).split(" "			)			)
													return bpe_tokens

						def 						a__ (					self		:			List[str]	,						_lowerCamelCase		:			List[str]			):
													return self.encoder.get(_lowerCamelCase	,						self.encoder.get(self.unk_token			)			)

						def 						a__ (					self		:			Any	,						_lowerCamelCase		:			int			):
													return self.decoder.get(_lowerCamelCase			)

						def 						a__ (					self		:			Any	,						_lowerCamelCase		:			Optional[Any]			):
													_UpperCAmelCase  :			str	       =		"".join(_lowerCamelCase			)
													_UpperCAmelCase  :			Dict	       =		bytearray([self.byte_decoder[c] for c in text]			).decode("utf-8"	,						errors=self.errors			)
													return text

						def 						a__ (					self		:			List[Any]	,						_lowerCamelCase		:			str	,						_lowerCamelCase		:			Optional[str] = None			):
													if not os.path.isdir(_lowerCamelCase			):
																				logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""			)
																				return
													_UpperCAmelCase  :			int	       =		os.path.join(
													    _lowerCamelCase	,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
													_UpperCAmelCase  :			Tuple	       =		os.path.join(
													    _lowerCamelCase	,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

													with open(_lowerCamelCase	,						"w"	,						encoding="utf-8"			) as f:
																				f.write(json.dumps(self.encoder	,						indent=2	,						sort_keys=_lowerCamelCase	,						ensure_ascii=_lowerCamelCase			) + "\n"			)

													_UpperCAmelCase  :			str	       =		0
													with open(_lowerCamelCase	,						"w"	,						encoding="utf-8"			) as writer:
																				writer.write("#version: 0.2\n"			)
																				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,						key=lambda _lowerCamelCase			: kv[1]			):
																											if index != token_index:
																																		logger.warning(
																																		    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																																		    " Please check that the tokenizer is not corrupted!"			)
																																		_UpperCAmelCase  :			Tuple	       =		token_index
																											writer.write(" ".join(_lowerCamelCase			) + "\n"			)
																											index += 1

													return vocab_file, merge_file

						def 						a__ (					self		:			Tuple	,						_lowerCamelCase		:			Any	,						_lowerCamelCase		:			Union[str, Any]=False	,						**_lowerCamelCase		:			Tuple			):
													_UpperCAmelCase  :			int	       =		kwargs.pop("add_prefix_space"	,						self.add_prefix_space			)
													if is_split_into_words or add_prefix_space:
																				_UpperCAmelCase  :			Dict	       =		" " + text
													return (text, kwargs)

						def 						a__ (					self		:			str	,						_lowerCamelCase		:			Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]	,						_lowerCamelCase		:			bool = False	,						_lowerCamelCase		:			bool = None	,						_lowerCamelCase		:			Optional[List[str]] = None	,						**_lowerCamelCase		:			Optional[int]	,						):
													_UpperCAmelCase  :			Optional[Any]	       =		super()._decode(
													    token_ids=_lowerCamelCase	,						skip_special_tokens=_lowerCamelCase	,						clean_up_tokenization_spaces=_lowerCamelCase	,						**_lowerCamelCase	,						)

													if truncate_before_pattern is not None and len(_lowerCamelCase			) > 0:
																				_UpperCAmelCase  :			Optional[Any]	       =		self.truncate(_lowerCamelCase	,						_lowerCamelCase			)

													return decoded_text






						def 						a__ (					self		:			Optional[Any]	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			List[Any]			):
													def find_re(_lowerCamelCase		:			Dict	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			List[str]			):
																				_UpperCAmelCase  :			Any	       =		pattern.search(_lowerCamelCase	,						_lowerCamelCase			)
																				return m.start() if m else -1

													_UpperCAmelCase  :			Union[str, Any]	       =		[re.compile(_lowerCamelCase	,						re.MULTILINE			) for pattern in truncate_before_pattern]

													_UpperCAmelCase  :			Tuple	       =		list(re.finditer("^print"	,						_lowerCamelCase	,						re.MULTILINE			)			)

													if len(_lowerCamelCase			) > 1:
																				_UpperCAmelCase  :			int	       =		completion[: prints[1].start()]

													_UpperCAmelCase  :			Optional[Any]	       =		list(re.finditer("^def"	,						_lowerCamelCase	,						re.MULTILINE			)			)

													if len(_lowerCamelCase			) > 1:
																				_UpperCAmelCase  :			Optional[Any]	       =		completion[: defs[1].start()]

													_UpperCAmelCase  :			List[Any]	       =		0

													_UpperCAmelCase  :			Optional[Any]	       =		[
													    pos for pos in [find_re(_lowerCamelCase	,						_lowerCamelCase	,						_lowerCamelCase			) for terminal in terminals] if pos != -1
													]

													if len(_lowerCamelCase			) > 0:
																				return completion[: min(_lowerCamelCase			)]
													else:
																				return completion






def 		lowerCAmelCase_     (     _SCREAMING_SNAKE_CASE      ,	_SCREAMING_SNAKE_CASE      ,	_SCREAMING_SNAKE_CASE  )   ->      Dict:

							"""simple docstring"""






							if power < 0:
														return -1
							base %= mod
							_UpperCAmelCase  :			Optional[Any]	       =		1

							while power > 0:
														if power & 1:
																					_UpperCAmelCase  :			Optional[Any]	       =		(result * base) % mod
														_UpperCAmelCase  :			List[str]	       =		power >> 1
														_UpperCAmelCase  :			Optional[int]	       =		(base * base) % mod

							return result




def 		lowerCAmelCase_     (     )   ->      List[str]:

							"""simple docstring"""






							print(modular_exponential(3      ,	2_0_0      ,	1_3  )  )


if __name__ == "__main__":
				import doctest

				doctest.testmod()

				main()




