



'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
							if is_torch_available():
														import torch
							if is_tf_available():
														import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


SCREAMING_SNAKE_CASE__     : Union[str, Any]				=   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     : List[Any]				=   {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

SCREAMING_SNAKE_CASE__     : str				=   {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

SCREAMING_SNAKE_CASE__     : List[Any]				=   {
    """Salesforce/codegen-350M-mono""": 20_48,
}







@lru_cache()
def  SCREAMING_SNAKE_CASE__ (				):
		SCREAMING_SNAKE_CASE_						:Any										=      (
		    list(range(ord('!'     ) ,		ord('~'     ) + 1     )     ) + list(range(ord('¡'     ) ,		ord('¬'     ) + 1     )     ) + list(range(ord('®'     ) ,		ord('ÿ'     ) + 1     )     )
		)
		SCREAMING_SNAKE_CASE_						:Any										=      bs[:]
		SCREAMING_SNAKE_CASE_						:List[str]										=      0
		for b in range(2**8     ):
				if b not in bs:
						bs.append(SCREAMING_SNAKE_CASE     )
						cs.append(2**8 + n     )
						n += 1
		SCREAMING_SNAKE_CASE_						:Optional[Any]										=      [chr(SCREAMING_SNAKE_CASE     ) for n in cs]
		return dict(zip(SCREAMING_SNAKE_CASE ,		SCREAMING_SNAKE_CASE     )     )







def  SCREAMING_SNAKE_CASE__ (				SCREAMING_SNAKE_CASE     ):
		SCREAMING_SNAKE_CASE_						:str										=      set()
		SCREAMING_SNAKE_CASE_						:Optional[Any]										=      word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)     )
				SCREAMING_SNAKE_CASE_						:int										=      char
		return pairs





class 				__lowerCAmelCase(							lowerCAmelCase__      ):
			__snake_case  :		Dict		=      VOCAB_FILES_NAMES
			__snake_case  :		List[Any]		=      PRETRAINED_VOCAB_FILES_MAP
			__snake_case  :		List[str]		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case  :		Union[str, Any]		=      ['input_ids', 'attention_mask']






			def __init__(   self  :							int      ,       SCREAMING_SNAKE_CASE  :							int      ,       SCREAMING_SNAKE_CASE  :							Any      ,       SCREAMING_SNAKE_CASE  :							Dict="replace"      ,       SCREAMING_SNAKE_CASE  :							int="<|endoftext|>"      ,       SCREAMING_SNAKE_CASE  :							Optional[int]="<|endoftext|>"      ,       SCREAMING_SNAKE_CASE  :							Dict="<|endoftext|>"      ,       SCREAMING_SNAKE_CASE  :							List[str]=None      ,       SCREAMING_SNAKE_CASE  :							str=False      ,       SCREAMING_SNAKE_CASE  :							Tuple=False      ,       **SCREAMING_SNAKE_CASE  :							List[Any]      ,       ):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:int										=      AddedToken(SCREAMING_SNAKE_CASE      ,       lstrip=SCREAMING_SNAKE_CASE      ,       rstrip=SCREAMING_SNAKE_CASE	) if isinstance(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	) else bos_token
					SCREAMING_SNAKE_CASE_						:Any										=      AddedToken(SCREAMING_SNAKE_CASE      ,       lstrip=SCREAMING_SNAKE_CASE      ,       rstrip=SCREAMING_SNAKE_CASE	) if isinstance(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	) else eos_token
					SCREAMING_SNAKE_CASE_						:Tuple										=      AddedToken(SCREAMING_SNAKE_CASE      ,       lstrip=SCREAMING_SNAKE_CASE      ,       rstrip=SCREAMING_SNAKE_CASE	) if isinstance(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	) else unk_token
					SCREAMING_SNAKE_CASE_						:Tuple										=      AddedToken(SCREAMING_SNAKE_CASE      ,       lstrip=SCREAMING_SNAKE_CASE      ,       rstrip=SCREAMING_SNAKE_CASE	) if isinstance(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	) else pad_token
					super().__init__(
					    errors=SCREAMING_SNAKE_CASE      ,       unk_token=SCREAMING_SNAKE_CASE      ,       bos_token=SCREAMING_SNAKE_CASE      ,       eos_token=SCREAMING_SNAKE_CASE      ,       pad_token=SCREAMING_SNAKE_CASE      ,       add_prefix_space=SCREAMING_SNAKE_CASE      ,       add_bos_token=SCREAMING_SNAKE_CASE      ,       **SCREAMING_SNAKE_CASE      ,       )
					SCREAMING_SNAKE_CASE_						:str										=      add_bos_token

					with open(SCREAMING_SNAKE_CASE      ,       encoding='utf-8'	) as vocab_handle:
							SCREAMING_SNAKE_CASE_						:List[Any]										=      json.load(SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:Tuple										=      {v: k for k, v in self.encoder.items()}
					SCREAMING_SNAKE_CASE_						:str										=      errors  # how to handle errors in decoding
					SCREAMING_SNAKE_CASE_						:Tuple										=      bytes_to_unicode()
					SCREAMING_SNAKE_CASE_						:Dict										=      {v: k for k, v in self.byte_encoder.items()}
					with open(SCREAMING_SNAKE_CASE      ,       encoding='utf-8'	) as merges_handle:
							SCREAMING_SNAKE_CASE_						:int										=      merges_handle.read().split('\n'	)[1:-1]
					SCREAMING_SNAKE_CASE_						:List[str]										=      [tuple(merge.split()	) for merge in bpe_merges]
					SCREAMING_SNAKE_CASE_						:Dict										=      dict(zip(SCREAMING_SNAKE_CASE      ,       range(len(SCREAMING_SNAKE_CASE	)	)	)	)
					SCREAMING_SNAKE_CASE_						:str										=      {}
					SCREAMING_SNAKE_CASE_						:List[str]										=      add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					SCREAMING_SNAKE_CASE_						:Optional[int]										=      re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'	)






			@property
			def 				_lowercase	(   self  :							Optional[Any]	):




					"""simple docstring"""







					return len(self.encoder	)






			def 				_lowercase	(   self  :							Optional[Any]	):




					"""simple docstring"""







					return dict(self.encoder      ,       **self.added_tokens_encoder	)






			def 				_lowercase	(   self  :							Tuple      ,       SCREAMING_SNAKE_CASE  :							Optional[int]	):




					"""simple docstring"""







					if token in self.cache:
							return self.cache[token]
					SCREAMING_SNAKE_CASE_						:int										=      tuple(SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:Any										=      get_pairs(SCREAMING_SNAKE_CASE	)

					if not pairs:
							return token

					while True:
							SCREAMING_SNAKE_CASE_						:str										=      min(SCREAMING_SNAKE_CASE      ,       key=lambda SCREAMING_SNAKE_CASE	: self.bpe_ranks.get(SCREAMING_SNAKE_CASE      ,       float('inf'	)	)	)
							if bigram not in self.bpe_ranks:
									break
							SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_						:Optional[Any]										=      bigram
							SCREAMING_SNAKE_CASE_						:Any										=      []
							SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      0
							while i < len(SCREAMING_SNAKE_CASE	):
									try:
											SCREAMING_SNAKE_CASE_						:Dict										=      word.index(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)
									except ValueError:
											new_word.extend(word[i:]	)
											break
									else:
											new_word.extend(word[i:j]	)
											SCREAMING_SNAKE_CASE_						:str										=      j

									if word[i] == first and i < len(SCREAMING_SNAKE_CASE	) - 1 and word[i + 1] == second:
											new_word.append(first + second	)
											i += 2
									else:
											new_word.append(word[i]	)
											i += 1
							SCREAMING_SNAKE_CASE_						:str										=      tuple(SCREAMING_SNAKE_CASE	)
							SCREAMING_SNAKE_CASE_						:List[Any]										=      new_word
							if len(SCREAMING_SNAKE_CASE	) == 1:
									break
							else:
									SCREAMING_SNAKE_CASE_						:List[str]										=      get_pairs(SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:Optional[int]										=      ' '.join(SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:str										=      word
					return word






			def 				_lowercase	(   self  :							Union[str, Any]      ,       SCREAMING_SNAKE_CASE  :							List[str]      ,       SCREAMING_SNAKE_CASE  :							List[Any]=None	):




					"""simple docstring"""







					if self.add_bos_token:
							SCREAMING_SNAKE_CASE_						:Optional[Any]										=      [self.bos_token_id]
					else:
							SCREAMING_SNAKE_CASE_						:List[str]										=      []

					SCREAMING_SNAKE_CASE_						:List[str]										=      bos_token_ids + token_ids_a

					if token_ids_a is None:
							return output

					return output + bos_token_ids + token_ids_a






			def 				_lowercase	(   self  :							Any      ,       SCREAMING_SNAKE_CASE  :							Optional[Any]	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:int										=      []
					for token in re.findall(self.pat      ,       SCREAMING_SNAKE_CASE	):
							SCREAMING_SNAKE_CASE_						:str										=      ''.join(
							    self.byte_encoder[b] for b in token.encode('utf-8'	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE	).split(' '	)	)
					return bpe_tokens






			def 				_lowercase	(   self  :							Optional[int]      ,       SCREAMING_SNAKE_CASE  :							Any	):




					"""simple docstring"""







					return self.encoder.get(SCREAMING_SNAKE_CASE      ,       self.encoder.get(self.unk_token	)	)






			def 				_lowercase	(   self  :							List[str]      ,       SCREAMING_SNAKE_CASE  :							Optional[int]	):




					"""simple docstring"""







					return self.decoder.get(SCREAMING_SNAKE_CASE	)






			def 				_lowercase	(   self  :							Optional[Any]      ,       SCREAMING_SNAKE_CASE  :							Optional[int]	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:int										=      ''.join(SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:List[str]										=      bytearray([self.byte_decoder[c] for c in text]	).decode('utf-8'      ,       errors=self.errors	)
					return text






			def 				_lowercase	(   self  :							Optional[Any]      ,       SCREAMING_SNAKE_CASE  :							str      ,       SCREAMING_SNAKE_CASE  :							Optional[str] = None	):




					"""simple docstring"""







					if not os.path.isdir(SCREAMING_SNAKE_CASE	):
							logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
							return
					SCREAMING_SNAKE_CASE_						:Optional[int]										=      os.path.join(
					    SCREAMING_SNAKE_CASE      ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
					SCREAMING_SNAKE_CASE_						:Tuple										=      os.path.join(
					    SCREAMING_SNAKE_CASE      ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

					with open(SCREAMING_SNAKE_CASE      ,       'w'      ,       encoding='utf-8'	) as f:
							f.write(json.dumps(self.encoder      ,       indent=2      ,       sort_keys=SCREAMING_SNAKE_CASE      ,       ensure_ascii=SCREAMING_SNAKE_CASE	) + '\n'	)

					SCREAMING_SNAKE_CASE_						:str										=      0
					with open(SCREAMING_SNAKE_CASE      ,       'w'      ,       encoding='utf-8'	) as writer:
							writer.write('#version: 0.2\n'	)
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,       key=lambda SCREAMING_SNAKE_CASE	: kv[1]	):
									if index != token_index:
											logger.warning(
											    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
											    ' Please check that the tokenizer is not corrupted!'	)
											SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      token_index
									writer.write(' '.join(SCREAMING_SNAKE_CASE	) + '\n'	)
									index += 1

					return vocab_file, merge_file






			def 				_lowercase	(   self  :							Optional[int]      ,       SCREAMING_SNAKE_CASE  :							Tuple      ,       SCREAMING_SNAKE_CASE  :							List[Any]=False      ,       **SCREAMING_SNAKE_CASE  :							Any	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Optional[int]										=      kwargs.pop('add_prefix_space'      ,       self.add_prefix_space	)
					if is_split_into_words or add_prefix_space:
							SCREAMING_SNAKE_CASE_						:Tuple										=      ' ' + text
					return (text, kwargs)






			def 				_lowercase	(   self  :							Any      ,       SCREAMING_SNAKE_CASE  :							Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]      ,       SCREAMING_SNAKE_CASE  :							bool = False      ,       SCREAMING_SNAKE_CASE  :							bool = None      ,       SCREAMING_SNAKE_CASE  :							Optional[List[str]] = None      ,       **SCREAMING_SNAKE_CASE  :							Any      ,       ):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:str										=      super()._decode(
					    token_ids=SCREAMING_SNAKE_CASE      ,       skip_special_tokens=SCREAMING_SNAKE_CASE      ,       clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE      ,       **SCREAMING_SNAKE_CASE      ,       )

					if truncate_before_pattern is not None and len(SCREAMING_SNAKE_CASE	) > 0:
							SCREAMING_SNAKE_CASE_						:Optional[int]										=      self.truncate(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)

					return decoded_text






			def 				_lowercase	(   self  :							Optional[int]      ,       SCREAMING_SNAKE_CASE  :							List[Any]      ,       SCREAMING_SNAKE_CASE  :							List[str]	):




					"""simple docstring"""







					def find_re(SCREAMING_SNAKE_CASE  :							Tuple      ,       SCREAMING_SNAKE_CASE  :							Tuple      ,       SCREAMING_SNAKE_CASE  :							Optional[Any]	):
							SCREAMING_SNAKE_CASE_						:Any										=      pattern.search(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)
							return m.start() if m else -1

					SCREAMING_SNAKE_CASE_						:Tuple										=      [re.compile(SCREAMING_SNAKE_CASE      ,       re.MULTILINE	) for pattern in truncate_before_pattern]

					SCREAMING_SNAKE_CASE_						:str										=      list(re.finditer('^print'      ,       SCREAMING_SNAKE_CASE      ,       re.MULTILINE	)	)

					if len(SCREAMING_SNAKE_CASE	) > 1:
							SCREAMING_SNAKE_CASE_						:int										=      completion[: prints[1].start()]

					SCREAMING_SNAKE_CASE_						:Optional[int]										=      list(re.finditer('^def'      ,       SCREAMING_SNAKE_CASE      ,       re.MULTILINE	)	)

					if len(SCREAMING_SNAKE_CASE	) > 1:
							SCREAMING_SNAKE_CASE_						:Dict										=      completion[: defs[1].start()]

					SCREAMING_SNAKE_CASE_						:Optional[Any]										=      0

					SCREAMING_SNAKE_CASE_						:Any										=      [
					    pos for pos in [find_re(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	) for terminal in terminals] if pos != -1
					]

					if len(SCREAMING_SNAKE_CASE	) > 0:
							return completion[: min(SCREAMING_SNAKE_CASE	)]
					else:
							return completion

'''simple docstring'''
def 				__UpperCamelCase (					__UpperCAmelCase  :					float ,   __UpperCAmelCase  :					float ,   __UpperCAmelCase  :					float			)							->	list[int]:
  if hue < 0 or hue > 360:
    raise Exception('''hue should be between 0 and 360'''			)

  if saturation < 0 or saturation > 1:
    raise Exception('''saturation should be between 0 and 1'''			)

  if value < 0 or value > 1:
    raise Exception('''value should be between 0 and 1'''			)

  lowercase				:							List[str]									=   value * saturation
  lowercase				:							Optional[Any]									=   hue / 60
  lowercase				:							Dict									=   chroma * (1 - abs(hue_section % 2 - 1			))
  lowercase				:							List[Any]									=   value - chroma

  if hue_section >= 0 and hue_section <= 1:
    lowercase				:							Optional[int]									=   round(255 * (chroma + match_value)			)
    lowercase				:							List[Any]									=   round(255 * (second_largest_component + match_value)			)
    lowercase				:							int									=   round(255 * (match_value)			)
  elif hue_section > 1 and hue_section <= 2:
    lowercase				:							Optional[Any]									=   round(255 * (second_largest_component + match_value)			)
    lowercase				:							Any									=   round(255 * (chroma + match_value)			)
    lowercase				:							Any									=   round(255 * (match_value)			)
  elif hue_section > 2 and hue_section <= 3:
    lowercase				:							Dict									=   round(255 * (match_value)			)
    lowercase				:							Optional[int]									=   round(255 * (chroma + match_value)			)
    lowercase				:							List[str]									=   round(255 * (second_largest_component + match_value)			)
  elif hue_section > 3 and hue_section <= 4:
    lowercase				:							List[str]									=   round(255 * (match_value)			)
    lowercase				:							List[str]									=   round(255 * (second_largest_component + match_value)			)
    lowercase				:							Tuple									=   round(255 * (chroma + match_value)			)
  elif hue_section > 4 and hue_section <= 5:
    lowercase				:							Optional[int]									=   round(255 * (second_largest_component + match_value)			)
    lowercase				:							Tuple									=   round(255 * (match_value)			)
    lowercase				:							Optional[int]									=   round(255 * (chroma + match_value)			)
  else:
    lowercase				:							int									=   round(255 * (chroma + match_value)			)
    lowercase				:							Optional[Any]									=   round(255 * (match_value)			)
    lowercase				:							Optional[int]									=   round(255 * (second_largest_component + match_value)			)

  return [red, green, blue]







def 				__UpperCamelCase (					__UpperCAmelCase  :					int ,   __UpperCAmelCase  :					int ,   __UpperCAmelCase  :					int			)							->	list[float]:
  if red < 0 or red > 255:
    raise Exception('''red should be between 0 and 255'''			)

  if green < 0 or green > 255:
    raise Exception('''green should be between 0 and 255'''			)

  if blue < 0 or blue > 255:
    raise Exception('''blue should be between 0 and 255'''			)

  lowercase				:							str									=   red / 255
  lowercase				:							Union[str, Any]									=   green / 255
  lowercase				:							Union[str, Any]									=   blue / 255
  lowercase				:							str									=   max(__UpperCAmelCase ,   __UpperCAmelCase ,   __UpperCAmelCase			)
  lowercase				:							int									=   value - min(__UpperCAmelCase ,   __UpperCAmelCase ,   __UpperCAmelCase			)
  lowercase				:							Optional[int]									=   0 if value == 0 else chroma / value

  if chroma == 0:
    lowercase				:							Dict									=   0.0
  elif value == float_red:
    lowercase				:							Tuple									=   60 * (0 + (float_green - float_blue) / chroma)
  elif value == float_green:
    lowercase				:							Optional[int]									=   60 * (2 + (float_blue - float_red) / chroma)
  else:
    lowercase				:							Dict									=   60 * (4 + (float_red - float_green) / chroma)

  lowercase				:							List[str]									=   (hue + 360) % 360

  return [hue, saturation, value]







def 				__UpperCamelCase (					__UpperCAmelCase  :					list[float] ,   __UpperCAmelCase  :					list[float]			)							->	bool:
  lowercase				:							int									=   abs(hsv_a[0] - hsv_a[0]			) < 0.2
  lowercase				:							Optional[int]									=   abs(hsv_a[1] - hsv_a[1]			) < 0.002
  lowercase				:							Dict									=   abs(hsv_a[2] - hsv_a[2]			) < 0.002

  return check_hue and check_saturation and check_value