


'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
							if is_torch_available():
														import torch
							if is_tf_available():
														import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__SCREAMING_SNAKE_CASE				:							Dict		       =		logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE				:							str		       =		{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__SCREAMING_SNAKE_CASE				:							Optional[Any]		       =		{
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

__SCREAMING_SNAKE_CASE				:							List[str]		       =		{
    '''Salesforce/codegen-350M-mono''': 2048,
}







@lru_cache()
def 			a_							(  ):
							A_ 						=		(
							    list(range(ord("!"   )	,     ord("~"   ) + 1   )   ) + list(range(ord("¡"   )	,     ord("¬"   ) + 1   )   ) + list(range(ord("®"   )	,     ord("ÿ"   ) + 1   )   )
							)
							A_ 						=		bs[:]
							A_ 						=		0
							for b in range(2**8   ):
														if b not in bs:
																					bs.append(UpperCamelCase_   )
																					cs.append(2**8 + n   )
																					n += 1
							A_ 						=		[chr(UpperCamelCase_   ) for n in cs]
							return dict(zip(UpperCamelCase_	,     UpperCamelCase_   )   )







def 			a_							(  UpperCamelCase_   ):
							A_ 						=		set()
							A_ 						=		word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)   )
														A_ 						=		char
							return pairs







class 				__lowerCAmelCase      (      lowercase	):

						"""simple docstring"""






						_UpperCAmelCase		:					Tuple      	=	VOCAB_FILES_NAMES
						_UpperCAmelCase		:					Union[str, Any]      	=	PRETRAINED_VOCAB_FILES_MAP
						_UpperCAmelCase		:					Optional[int]      	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_UpperCAmelCase		:					Optional[Any]      	=	["input_ids", "attention_mask"]






						def __init__(     self       :			Union[str, Any]   ,     lowerCAmelCase       :			Optional[int]   ,     lowerCAmelCase       :			List[str]   ,     lowerCAmelCase       :			str="replace"   ,     lowerCAmelCase       :			int="<|endoftext|>"   ,     lowerCAmelCase       :			int="<|endoftext|>"   ,     lowerCAmelCase       :			str="<|endoftext|>"   ,     lowerCAmelCase       :			Optional[int]=None   ,     lowerCAmelCase       :			Any=False   ,     lowerCAmelCase       :			Any=False   ,     **lowerCAmelCase       :			Union[str, Any]   ,     ):
													A_ 						=		AddedToken(lowerCAmelCase   ,     lstrip=lowerCAmelCase   ,     rstrip=lowerCAmelCase  ) if isinstance(lowerCAmelCase   ,     lowerCAmelCase  ) else bos_token
													A_ 						=		AddedToken(lowerCAmelCase   ,     lstrip=lowerCAmelCase   ,     rstrip=lowerCAmelCase  ) if isinstance(lowerCAmelCase   ,     lowerCAmelCase  ) else eos_token
													A_ 						=		AddedToken(lowerCAmelCase   ,     lstrip=lowerCAmelCase   ,     rstrip=lowerCAmelCase  ) if isinstance(lowerCAmelCase   ,     lowerCAmelCase  ) else unk_token
													A_ 						=		AddedToken(lowerCAmelCase   ,     lstrip=lowerCAmelCase   ,     rstrip=lowerCAmelCase  ) if isinstance(lowerCAmelCase   ,     lowerCAmelCase  ) else pad_token
													super().__init__(
													    errors=lowerCAmelCase   ,     unk_token=lowerCAmelCase   ,     bos_token=lowerCAmelCase   ,     eos_token=lowerCAmelCase   ,     pad_token=lowerCAmelCase   ,     add_prefix_space=lowerCAmelCase   ,     add_bos_token=lowerCAmelCase   ,     **lowerCAmelCase   ,     )
													A_ 						=		add_bos_token

													with open(lowerCAmelCase   ,     encoding="utf-8"  ) as vocab_handle:
																				A_ 						=		json.load(lowerCAmelCase  )
													A_ 						=		{v: k for k, v in self.encoder.items()}
													A_ 						=		errors  # how to handle errors in decoding
													A_ 						=		bytes_to_unicode()
													A_ 						=		{v: k for k, v in self.byte_encoder.items()}
													with open(lowerCAmelCase   ,     encoding="utf-8"  ) as merges_handle:
																				A_ 						=		merges_handle.read().split("\n"  )[1:-1]
													A_ 						=		[tuple(merge.split()  ) for merge in bpe_merges]
													A_ 						=		dict(zip(lowerCAmelCase   ,     range(len(lowerCAmelCase  )  )  )  )
													A_ 						=		{}
													A_ 						=		add_prefix_space

													# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
													A_ 						=		re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"  )






						@property
						def 							_UpperCAmelCase					(     self       :			Optional[int]  ):
													return len(self.encoder  )






						def 							_UpperCAmelCase					(     self       :			Union[str, Any]  ):
													return dict(self.encoder   ,     **self.added_tokens_encoder  )






						def 							_UpperCAmelCase					(     self       :			Any   ,     lowerCAmelCase       :			str  ):
													if token in self.cache:
																				return self.cache[token]
													A_ 						=		tuple(lowerCAmelCase  )
													A_ 						=		get_pairs(lowerCAmelCase  )

													if not pairs:
																				return token

													while True:
																				A_ 						=		min(lowerCAmelCase   ,     key=lambda lowerCAmelCase  : self.bpe_ranks.get(lowerCAmelCase   ,     float("inf"  )  )  )
																				if bigram not in self.bpe_ranks:
																											break
																				A_						,				A_ 						=		bigram
																				A_ 						=		[]
																				A_ 						=		0
																				while i < len(lowerCAmelCase  ):
																											try:
																																		A_ 						=		word.index(lowerCAmelCase   ,     lowerCAmelCase  )
																											except ValueError:
																																		new_word.extend(word[i:]  )
																																		break
																											else:
																																		new_word.extend(word[i:j]  )
																																		A_ 						=		j

																											if word[i] == first and i < len(lowerCAmelCase  ) - 1 and word[i + 1] == second:
																																		new_word.append(first + second  )
																																		i += 2
																											else:
																																		new_word.append(word[i]  )
																																		i += 1
																				A_ 						=		tuple(lowerCAmelCase  )
																				A_ 						=		new_word
																				if len(lowerCAmelCase  ) == 1:
																											break
																				else:
																											A_ 						=		get_pairs(lowerCAmelCase  )
													A_ 						=		" ".join(lowerCAmelCase  )
													A_ 						=		word
													return word






						def 							_UpperCAmelCase					(     self       :			str   ,     lowerCAmelCase       :			Tuple   ,     lowerCAmelCase       :			Any=None  ):
													if self.add_bos_token:
																				A_ 						=		[self.bos_token_id]
													else:
																				A_ 						=		[]

													A_ 						=		bos_token_ids + token_ids_a

													if token_ids_a is None:
																				return output

													return output + bos_token_ids + token_ids_a






						def 							_UpperCAmelCase					(     self       :			List[str]   ,     lowerCAmelCase       :			Any  ):
													A_ 						=		[]
													for token in re.findall(self.pat   ,     lowerCAmelCase  ):
																				A_ 						=		"".join(
																				    self.byte_encoder[b] for b in token.encode("utf-8"  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																				bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase  ).split(" "  )  )
													return bpe_tokens






						def 							_UpperCAmelCase					(     self       :			Union[str, Any]   ,     lowerCAmelCase       :			Optional[Any]  ):
													return self.encoder.get(lowerCAmelCase   ,     self.encoder.get(self.unk_token  )  )






						def 							_UpperCAmelCase					(     self       :			Optional[int]   ,     lowerCAmelCase       :			Optional[Any]  ):
													return self.decoder.get(lowerCAmelCase  )






						def 							_UpperCAmelCase					(     self       :			Optional[Any]   ,     lowerCAmelCase       :			List[Any]  ):
													A_ 						=		"".join(lowerCAmelCase  )
													A_ 						=		bytearray([self.byte_decoder[c] for c in text]  ).decode("utf-8"   ,     errors=self.errors  )
													return text






						def 							_UpperCAmelCase					(     self       :			Any   ,     lowerCAmelCase       :			str   ,     lowerCAmelCase       :			Optional[str] = None  ):
													if not os.path.isdir(lowerCAmelCase  ):
																				logger.error(F"Vocabulary path ({save_directory}) should be a directory"  )
																				return
													A_ 						=		os.path.join(
													    lowerCAmelCase   ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )
													A_ 						=		os.path.join(
													    lowerCAmelCase   ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]  )

													with open(lowerCAmelCase   ,     "w"   ,     encoding="utf-8"  ) as f:
																				f.write(json.dumps(self.encoder   ,     indent=2   ,     sort_keys=lowerCAmelCase   ,     ensure_ascii=lowerCAmelCase  ) + "\n"  )

													A_ 						=		0
													with open(lowerCAmelCase   ,     "w"   ,     encoding="utf-8"  ) as writer:
																				writer.write("#version: 0.2\n"  )
																				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,     key=lambda lowerCAmelCase  : kv[1]  ):
																											if index != token_index:
																																		logger.warning(
																																		    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																																		    " Please check that the tokenizer is not corrupted!"  )
																																		A_ 						=		token_index
																											writer.write(" ".join(lowerCAmelCase  ) + "\n"  )
																											index += 1

													return vocab_file, merge_file






						def 							_UpperCAmelCase					(     self       :			List[str]   ,     lowerCAmelCase       :			List[Any]   ,     lowerCAmelCase       :			Any=False   ,     **lowerCAmelCase       :			Union[str, Any]  ):
													A_ 						=		kwargs.pop("add_prefix_space"   ,     self.add_prefix_space  )
													if is_split_into_words or add_prefix_space:
																				A_ 						=		" " + text
													return (text, kwargs)






						def 							_UpperCAmelCase					(     self       :			int   ,     lowerCAmelCase       :			Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]   ,     lowerCAmelCase       :			bool = False   ,     lowerCAmelCase       :			bool = None   ,     lowerCAmelCase       :			Optional[List[str]] = None   ,     **lowerCAmelCase       :			int   ,     ):
													A_ 						=		super()._decode(
													    token_ids=lowerCAmelCase   ,     skip_special_tokens=lowerCAmelCase   ,     clean_up_tokenization_spaces=lowerCAmelCase   ,     **lowerCAmelCase   ,     )

													if truncate_before_pattern is not None and len(lowerCAmelCase  ) > 0:
																				A_ 						=		self.truncate(lowerCAmelCase   ,     lowerCAmelCase  )

													return decoded_text






						def 							_UpperCAmelCase					(     self       :			List[Any]   ,     lowerCAmelCase       :			List[str]   ,     lowerCAmelCase       :			int  ):
													def find_re(lowerCAmelCase       :			List[Any]   ,     lowerCAmelCase       :			Optional[Any]   ,     lowerCAmelCase       :			Optional[Any]  ):
																				A_ 						=		pattern.search(lowerCAmelCase   ,     lowerCAmelCase  )
																				return m.start() if m else -1

													A_ 						=		[re.compile(lowerCAmelCase   ,     re.MULTILINE  ) for pattern in truncate_before_pattern]

													A_ 						=		list(re.finditer("^print"   ,     lowerCAmelCase   ,     re.MULTILINE  )  )

													if len(lowerCAmelCase  ) > 1:
																				A_ 						=		completion[: prints[1].start()]

													A_ 						=		list(re.finditer("^def"   ,     lowerCAmelCase   ,     re.MULTILINE  )  )

													if len(lowerCAmelCase  ) > 1:
																				A_ 						=		completion[: defs[1].start()]

													A_ 						=		0

													A_ 						=		[
													    pos for pos in [find_re(lowerCAmelCase   ,     lowerCAmelCase   ,     lowerCAmelCase  ) for terminal in terminals] if pos != -1
													]

													if len(lowerCAmelCase  ) > 0:
																				return completion[: min(lowerCAmelCase  )]
													else:
																				return completion









'''simple docstring'''
def 			a_							(  UpperCamelCase_	,     UpperCamelCase_ = 1_0_0_0   ):
							if (not isinstance(UpperCamelCase_	,     UpperCamelCase_   )) or (digit_position <= 0):
														raise ValueError("Digit position must be a positive integer"   )
							elif (not isinstance(UpperCamelCase_	,     UpperCamelCase_   )) or (precision < 0):
														raise ValueError("Precision must be a nonnegative integer"   )

							# compute an approximation of (16 ** (n - 1)) * pi whose fractional part is mostly
							# accurate
							A_ 						=		(
							    4 * _subsum(UpperCamelCase_	,     1	,     UpperCamelCase_   )
							    - 2 * _subsum(UpperCamelCase_	,     4	,     UpperCamelCase_   )
							    - _subsum(UpperCamelCase_	,     5	,     UpperCamelCase_   )
							    - _subsum(UpperCamelCase_	,     6	,     UpperCamelCase_   )
							)

							# return the first hex digit of the fractional part of the result
							return hex(int((sum_result % 1) * 1_6   )   )[2:]







def 			a_							(  UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_   ):
							A_ 						=		0.0
							for sum_index in range(digit_pos_to_extract + precision   ):
														A_ 						=		8 * sum_index + denominator_addend
														if sum_index < digit_pos_to_extract:
																					# if the exponential term is an integer and we mod it by the denominator
																					# before dividing, only the integer part of the sum will change;
																					# the fractional part will not
																					A_ 						=		pow(
																					    1_6	,     digit_pos_to_extract - 1 - sum_index	,     UpperCamelCase_   )
														else:
																					A_ 						=		pow(1_6	,     digit_pos_to_extract - 1 - sum_index   )
														total += exponential_term / denominator
							return total


if __name__ == "__main__":
							import doctest

							doctest.testmod()





