"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__UpperCamelCase       :    str         =		logging.get_logger(__name__)

__UpperCamelCase       :    int         =		{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__UpperCamelCase       :    List[Any]         =		{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
}

__UpperCamelCase       :    Union[str, Any]         =		{
    "Salesforce/codegen-350M-mono": 2_0_4_8,
}


@lru_cache()
def   __UpperCAmelCase     (	):
   _lowercase							     =       (
       list(range(ord("!"       ),				ord("~"       ) + 1       )       ) + list(range(ord("¡"       ),				ord("¬"       ) + 1       )       ) + list(range(ord("®"       ),				ord("ÿ"       ) + 1       )       )
   )
   _lowercase							     =       bs[:]
   _lowercase							     =       0
   for b in range(2**8       ):
      if b not in bs:
         bs.append(a_       )
         cs.append(2**8 + n       )
         n += 1
   _lowercase							     =       [chr(a_       ) for n in cs]
   return dict(zip(a_,				a_       )       )


def   __UpperCAmelCase     (	_snake_case		:       Dict       ):
   _lowercase							     =       set()
   _lowercase							     =       word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)       )
      _lowercase							     =       char
   return pairs
class   UpperCAmelCase_	(				lowerCAmelCase__   ):
    snake_case_											=      VOCAB_FILES_NAMES
    snake_case_											=      PRETRAINED_VOCAB_FILES_MAP
    snake_case_											=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    snake_case_											=      ["""input_ids""", """attention_mask"""]
    def __init__(	self			:       Optional[int]   ,						_lowercase			:       List[Any]   ,						_lowercase			:       int   ,						_lowercase			:       Tuple="replace"   ,						_lowercase			:       List[str]="<|endoftext|>"   ,						_lowercase			:       Tuple="<|endoftext|>"   ,						_lowercase			:       Union[str, Any]="<|endoftext|>"   ,						_lowercase			:       int=None   ,						_lowercase			:       Optional[int]=False   ,						_lowercase			:       Tuple=False   ,						**_lowercase			:       Optional[Any]   ,						)						->    Union[str, Any]:
       _lowercase							     =       AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__		) if isinstance(snake_case__   ,						snake_case__		) else bos_token
       _lowercase							     =       AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__		) if isinstance(snake_case__   ,						snake_case__		) else eos_token
       _lowercase							     =       AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__		) if isinstance(snake_case__   ,						snake_case__		) else unk_token
       _lowercase							     =       AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__		) if isinstance(snake_case__   ,						snake_case__		) else pad_token
       super().__init__(
           errors=snake_case__   ,						unk_token=snake_case__   ,						bos_token=snake_case__   ,						eos_token=snake_case__   ,						pad_token=snake_case__   ,						add_prefix_space=snake_case__   ,						add_bos_token=snake_case__   ,						**snake_case__   ,						)
       _lowercase							     =       add_bos_token

       with open(snake_case__   ,						encoding="utf-8"		) as vocab_handle:
          _lowercase							     =       json.load(snake_case__		)
       _lowercase							     =       {v: k for k, v in self.encoder.items()}
       _lowercase							     =       errors  # how to handle errors in decoding
       _lowercase							     =       bytes_to_unicode()
       _lowercase							     =       {v: k for k, v in self.byte_encoder.items()}
       with open(snake_case__   ,						encoding="utf-8"		) as merges_handle:
          _lowercase							     =       merges_handle.read().split("\n"		)[1:-1]
       _lowercase							     =       [tuple(merge.split()		) for merge in bpe_merges]
       _lowercase							     =       dict(zip(snake_case__   ,						range(len(snake_case__		)		)		)		)
       _lowercase							     =       {}
       _lowercase							     =       add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       _lowercase							     =       re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"		)
    @property
    def 		_lowerCamelCase      (	self			:       List[Any]		)						->    List[Any]:
       return len(self.encoder		)
    def 		_lowerCamelCase      (	self			:       Tuple		)						->    int:
       return dict(self.encoder   ,						**self.added_tokens_encoder		)
    def 		_lowerCamelCase      (	self			:       str   ,						_lowercase			:       Union[str, Any]		)						->    Tuple:





       if token in self.cache:
          return self.cache[token]
       _lowercase							     =       tuple(snake_case__		)
       _lowercase							     =       get_pairs(snake_case__		)

       if not pairs:
          return token

       while True:
          _lowercase							     =       min(snake_case__   ,						key=lambda _lowercase		: self.bpe_ranks.get(snake_case__   ,						float("inf"		)		)		)
          if bigram not in self.bpe_ranks:
             break
          _lowercase     ,  _lowercase							     =       bigram
          _lowercase							     =       []
          _lowercase							     =       0
          while i < len(snake_case__		):
             try:
                _lowercase							     =       word.index(snake_case__   ,						snake_case__		)
             except ValueError:
                new_word.extend(word[i:]		)
                break
             else:
                new_word.extend(word[i:j]		)
                _lowercase							     =       j

             if word[i] == first and i < len(snake_case__		) - 1 and word[i + 1] == second:
                new_word.append(first + second		)
                i += 2
             else:
                new_word.append(word[i]		)
                i += 1
          _lowercase							     =       tuple(snake_case__		)
          _lowercase							     =       new_word
          if len(snake_case__		) == 1:
             break
          else:
             _lowercase							     =       get_pairs(snake_case__		)
       _lowercase							     =       " ".join(snake_case__		)
       _lowercase							     =       word
       return word
    def 		_lowerCamelCase      (	self			:       Optional[Any]   ,						_lowercase			:       Any   ,						_lowercase			:       Optional[Any]=None		)						->    Optional[Any]:





       if self.add_bos_token:
          _lowercase							     =       [self.bos_token_id]
       else:
          _lowercase							     =       []

       _lowercase							     =       bos_token_ids + token_ids_a

       if token_ids_a is None:
          return output

       return output + bos_token_ids + token_ids_a
    def 		_lowerCamelCase      (	self			:       List[str]   ,						_lowercase			:       List[Any]		)						->    Optional[Any]:
       _lowercase							     =       []
       for token in re.findall(self.pat   ,						snake_case__		):
          _lowercase							     =       "".join(
              self.byte_encoder[b] for b in token.encode("utf-8"		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
          bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case__		).split(" "		)		)
       return bpe_tokens
    def 		_lowerCamelCase      (	self			:       Tuple   ,						_lowercase			:       List[Any]		)						->    int:
       return self.encoder.get(snake_case__   ,						self.encoder.get(self.unk_token		)		)
    def 		_lowerCamelCase      (	self			:       str   ,						_lowercase			:       Dict		)						->    Optional[int]:
       return self.decoder.get(snake_case__		)
    def 		_lowerCamelCase      (	self			:       Union[str, Any]   ,						_lowercase			:       List[Any]		)						->    Optional[int]:
       _lowercase							     =       "".join(snake_case__		)
       _lowercase							     =       bytearray([self.byte_decoder[c] for c in text]		).decode("utf-8"   ,						errors=self.errors		)
       return text
    def 		_lowerCamelCase      (	self			:       List[str]   ,						_lowercase			:       str   ,						_lowercase			:       Optional[str] = None		)						->    Any:





       if not os.path.isdir(snake_case__		):
          logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""		)
          return
       _lowercase							     =       os.path.join(
           snake_case__   ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
       _lowercase							     =       os.path.join(
           snake_case__   ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

       with open(snake_case__   ,						"w"   ,						encoding="utf-8"		) as f:
          f.write(json.dumps(self.encoder   ,						indent=2   ,						sort_keys=snake_case__   ,						ensure_ascii=snake_case__		) + "\n"		)

       _lowercase							     =       0
       with open(snake_case__   ,						"w"   ,						encoding="utf-8"		) as writer:
          writer.write("#version: 0.2\n"		)
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,						key=lambda _lowercase		: kv[1]		):
             if index != token_index:
                logger.warning(
                    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                    " Please check that the tokenizer is not corrupted!"		)
                _lowercase							     =       token_index
             writer.write(" ".join(snake_case__		) + "\n"		)
             index += 1

       return vocab_file, merge_file
    def 		_lowerCamelCase      (	self			:       str   ,						_lowercase			:       Optional[int]   ,						_lowercase			:       Optional[int]=False   ,						**_lowercase			:       List[Any]		)						->    Tuple:
       _lowercase							     =       kwargs.pop("add_prefix_space"   ,						self.add_prefix_space		)
       if is_split_into_words or add_prefix_space:
          _lowercase							     =       " " + text
       return (text, kwargs)
    def 		_lowerCamelCase      (	self			:       str   ,						_lowercase			:       Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]   ,						_lowercase			:       bool = False   ,						_lowercase			:       bool = None   ,						_lowercase			:       Optional[List[str]] = None   ,						**_lowercase			:       Union[str, Any]   ,						)						->    Optional[int]:
       _lowercase							     =       super()._decode(
           token_ids=snake_case__   ,						skip_special_tokens=snake_case__   ,						clean_up_tokenization_spaces=snake_case__   ,						**snake_case__   ,						)

       if truncate_before_pattern is not None and len(snake_case__		) > 0:
          _lowercase							     =       self.truncate(snake_case__   ,						snake_case__		)

       return decoded_text




    def 		_lowerCamelCase      (	self			:       Tuple   ,						_lowercase			:       List[Any]   ,						_lowercase			:       Union[str, Any]		)						->    List[str]:





       def find_re(_lowercase			:       Dict   ,						_lowercase			:       Tuple   ,						_lowercase			:       int		):
          _lowercase							     =       pattern.search(snake_case__   ,						snake_case__		)
          return m.start() if m else -1

       _lowercase							     =       [re.compile(snake_case__   ,						re.MULTILINE		) for pattern in truncate_before_pattern]

       _lowercase							     =       list(re.finditer("^print"   ,						snake_case__   ,						re.MULTILINE		)		)

       if len(snake_case__		) > 1:
          _lowercase							     =       completion[: prints[1].start()]

       _lowercase							     =       list(re.finditer("^def"   ,						snake_case__   ,						re.MULTILINE		)		)

       if len(snake_case__		) > 1:
          _lowercase							     =       completion[: defs[1].start()]

       _lowercase							     =       0

       _lowercase							     =       [
           pos for pos in [find_re(snake_case__   ,						snake_case__   ,						snake_case__		) for terminal in terminals] if pos != -1
       ]

       if len(snake_case__		) > 0:
          return completion[: min(snake_case__		)]
       else:
          return completion

from __future__ import annotations

from typing import Any

class        UpperCamelCase__:

	"""simple docstring"""

	def __init__(      self				:			Dict							,			snake_case__				:			Any							,			snake_case__				:			Any ):  # noqa: A002
			"""simple docstring"""





			A				=item
			A				=next


class        UpperCamelCase__:

	"""simple docstring"""

	def __init__(      self				:			Optional[Any] ):
			"""simple docstring"""





			A				=None
			A				=0







	def        _a					(      self				:			str							,			snake_case__				:			Any ):
			"""simple docstring"""





			A				=Node(snake_case__							,			self.head )
			self.size += 1







	def        _a					(      self				:			Dict ):
			"""simple docstring"""





			if self.head is None:
					return None
			else:
					A				=self.head.item
					A				=self.head.next
					self.size -= 1
					return item







	def        _a					(      self				:			Any ):
			"""simple docstring"""





			return self.head is None







	def __str__(      self				:			Dict ):
			"""simple docstring"""





			if self.is_empty():
					return ""
			else:
					A				=self.head
					A				=""
					A				=[]
					while iterate:
							item_list.append(str(iterate.item ) )
							A				=iterate.next

					A				=" --> ".join(snake_case__ )

					return item_str







	def __len__(      self				:			Union[str, Any] ):
			"""simple docstring"""





			return self.size




