





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
							if is_torch_available():
														import torch
							if is_tf_available():
														import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


UpperCAmelCase__    :      Dict       					=logging.get_logger(__name__)

UpperCAmelCase__    :      Any       					={
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

UpperCAmelCase__    :      Dict       					={
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

UpperCAmelCase__    :      Optional[int]       					={
    '''Salesforce/codegen-350M-mono''': 20_48,
}





@lru_cache()
def   _lowercase       (    )    ->	List[str]:
				lowerCamelCase									=(
				    list(range(ord("""!"""					)      ,   ord("""~"""					) + 1					)					) + list(range(ord("""¡"""					)      ,   ord("""¬"""					) + 1					)					) + list(range(ord("""®"""					)      ,   ord("""ÿ"""					) + 1					)					)
				)
				lowerCamelCase									=bs[:]
				lowerCamelCase									=0
				for b in range(2**8					):
								if b not in bs:
												bs.append(_UpperCAmelCase					)
												cs.append(2**8 + n					)
												n += 1
				lowerCamelCase									=[chr(_UpperCAmelCase					) for n in cs]
				return dict(zip(_UpperCAmelCase      ,   _UpperCAmelCase					)					)





def   _lowercase       (    _UpperCAmelCase					)    ->	Union[str, Any]:
				lowerCamelCase									=set()
				lowerCamelCase									=word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)					)
								lowerCamelCase									=char
				return pairs




class    __A  (    a     ):
				__A  =					VOCAB_FILES_NAMES
				__A  =					PRETRAINED_VOCAB_FILES_MAP
				__A  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A  =					["""input_ids""", """attention_mask"""]


				def __init__(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ ,			UpperCAmelCase_="replace" ,			UpperCAmelCase_="<|endoftext|>" ,			UpperCAmelCase_="<|endoftext|>" ,			UpperCAmelCase_="<|endoftext|>" ,			UpperCAmelCase_=None ,			UpperCAmelCase_=False ,			UpperCAmelCase_=False ,			**UpperCAmelCase_ ,			):
								lowerCamelCase									=AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_				) else bos_token
								lowerCamelCase									=AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_				) else eos_token
								lowerCamelCase									=AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_				) else unk_token
								lowerCamelCase									=AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_				) else pad_token
								super().__init__(
								    errors=UpperCAmelCase_ ,			unk_token=UpperCAmelCase_ ,			bos_token=UpperCAmelCase_ ,			eos_token=UpperCAmelCase_ ,			pad_token=UpperCAmelCase_ ,			add_prefix_space=UpperCAmelCase_ ,			add_bos_token=UpperCAmelCase_ ,			**UpperCAmelCase_ ,			)
								lowerCamelCase									=add_bos_token

								with open(UpperCAmelCase_ ,			encoding="""utf-8"""				) as vocab_handle:
												lowerCamelCase									=json.load(UpperCAmelCase_				)
								lowerCamelCase									={v: k for k, v in self.encoder.items()}
								lowerCamelCase									=errors  # how to handle errors in decoding
								lowerCamelCase									=bytes_to_unicode()
								lowerCamelCase									={v: k for k, v in self.byte_encoder.items()}
								with open(UpperCAmelCase_ ,			encoding="""utf-8"""				) as merges_handle:
												lowerCamelCase									=merges_handle.read().split("""\n"""				)[1:-1]
								lowerCamelCase									=[tuple(merge.split()				) for merge in bpe_merges]
								lowerCamelCase									=dict(zip(UpperCAmelCase_ ,			range(len(UpperCAmelCase_				)				)				)				)
								lowerCamelCase									={}
								lowerCamelCase									=add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								lowerCamelCase									=re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""				)


				@property
				def _snake_case							(		self				):
								return len(self.encoder				)


				def _snake_case							(		self				):
								return dict(self.encoder ,			**self.added_tokens_encoder				)


				def _snake_case							(		self ,			UpperCAmelCase_				):
								if token in self.cache:
												return self.cache[token]
								lowerCamelCase									=tuple(UpperCAmelCase_				)
								lowerCamelCase									=get_pairs(UpperCAmelCase_				)

								if not pairs:
												return token

								while True:
												lowerCamelCase									=min(UpperCAmelCase_ ,			key=lambda UpperCAmelCase_				: self.bpe_ranks.get(UpperCAmelCase_ ,			float("""inf"""				)				)				)
												if bigram not in self.bpe_ranks:
																break
												lowerCamelCase			,	lowerCamelCase									=bigram
												lowerCamelCase									=[]
												lowerCamelCase									=0
												while i < len(UpperCAmelCase_				):
																try:
																				lowerCamelCase									=word.index(UpperCAmelCase_ ,			UpperCAmelCase_				)
																except ValueError:
																				new_word.extend(word[i:]				)
																				break
																else:
																				new_word.extend(word[i:j]				)
																				lowerCamelCase									=j

																if word[i] == first and i < len(UpperCAmelCase_				) - 1 and word[i + 1] == second:
																				new_word.append(first + second				)
																				i += 2
																else:
																				new_word.append(word[i]				)
																				i += 1
												lowerCamelCase									=tuple(UpperCAmelCase_				)
												lowerCamelCase									=new_word
												if len(UpperCAmelCase_				) == 1:
																break
												else:
																lowerCamelCase									=get_pairs(UpperCAmelCase_				)
								lowerCamelCase									=""" """.join(UpperCAmelCase_				)
								lowerCamelCase									=word
								return word


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_=None				):
								if self.add_bos_token:
												lowerCamelCase									=[self.bos_token_id]
								else:
												lowerCamelCase									=[]

								lowerCamelCase									=bos_token_ids + token_ids_a

								if token_ids_a is None:
												return output

								return output + bos_token_ids + token_ids_a


				def _snake_case							(		self ,			UpperCAmelCase_				):
								lowerCamelCase									=[]
								for token in re.findall(self.pat ,			UpperCAmelCase_				):
												lowerCamelCase									="""""".join(
												    self.byte_encoder[b] for b in token.encode("""utf-8"""				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
												bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase_				).split(""" """				)				)
								return bpe_tokens


				def _snake_case							(		self ,			UpperCAmelCase_				):
								return self.encoder.get(UpperCAmelCase_ ,			self.encoder.get(self.unk_token				)				)


				def _snake_case							(		self ,			UpperCAmelCase_				):
								return self.decoder.get(UpperCAmelCase_				)


				def _snake_case							(		self ,			UpperCAmelCase_				):
								lowerCamelCase									="""""".join(UpperCAmelCase_				)
								lowerCamelCase									=bytearray([self.byte_decoder[c] for c in text]				).decode("""utf-8""" ,			errors=self.errors				)
								return text


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ = None				):
								if not os.path.isdir(UpperCAmelCase_				):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
												return
								lowerCamelCase									=os.path.join(
								    UpperCAmelCase_ ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
								lowerCamelCase									=os.path.join(
								    UpperCAmelCase_ ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

								with open(UpperCAmelCase_ ,			"""w""" ,			encoding="""utf-8"""				) as f:
												f.write(json.dumps(self.encoder ,			indent=2 ,			sort_keys=UpperCAmelCase_ ,			ensure_ascii=UpperCAmelCase_				) + """\n"""				)

								lowerCamelCase									=0
								with open(UpperCAmelCase_ ,			"""w""" ,			encoding="""utf-8"""				) as writer:
												writer.write("""#version: 0.2\n"""				)
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,			key=lambda UpperCAmelCase_				: kv[1]				):
																if index != token_index:
																				logger.warning(
																				    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																				    """ Please check that the tokenizer is not corrupted!"""				)
																				lowerCamelCase									=token_index
																writer.write(""" """.join(UpperCAmelCase_				) + """\n"""				)
																index += 1

								return vocab_file, merge_file


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_=False ,			**UpperCAmelCase_				):
								lowerCamelCase									=kwargs.pop("""add_prefix_space""" ,			self.add_prefix_space				)
								if is_split_into_words or add_prefix_space:
												lowerCamelCase									=""" """ + text
								return (text, kwargs)


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ = False ,			UpperCAmelCase_ = None ,			UpperCAmelCase_ = None ,			**UpperCAmelCase_ ,			):
								lowerCamelCase									=super()._decode(
								    token_ids=UpperCAmelCase_ ,			skip_special_tokens=UpperCAmelCase_ ,			clean_up_tokenization_spaces=UpperCAmelCase_ ,			**UpperCAmelCase_ ,			)

								if truncate_before_pattern is not None and len(UpperCAmelCase_				) > 0:
												lowerCamelCase									=self.truncate(UpperCAmelCase_ ,			UpperCAmelCase_				)

								return decoded_text


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_				):
								def find_re(UpperCAmelCase_ ,			UpperCAmelCase_ ,			UpperCAmelCase_				):
												lowerCamelCase									=pattern.search(UpperCAmelCase_ ,			UpperCAmelCase_				)
												return m.start() if m else -1

								lowerCamelCase									=[re.compile(UpperCAmelCase_ ,			re.MULTILINE				) for pattern in truncate_before_pattern]

								lowerCamelCase									=list(re.finditer("""^print""" ,			UpperCAmelCase_ ,			re.MULTILINE				)				)

								if len(UpperCAmelCase_				) > 1:
												lowerCamelCase									=completion[: prints[1].start()]

								lowerCamelCase									=list(re.finditer("""^def""" ,			UpperCAmelCase_ ,			re.MULTILINE				)				)

								if len(UpperCAmelCase_				) > 1:
												lowerCamelCase									=completion[: defs[1].start()]

								lowerCamelCase									=0

								lowerCamelCase									=[
								    pos for pos in [find_re(UpperCAmelCase_ ,			UpperCAmelCase_ ,			UpperCAmelCase_				) for terminal in terminals] if pos != -1
								]

								if len(UpperCAmelCase_				) > 0:
												return completion[: min(UpperCAmelCase_				)]
								else:
												return completion












from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCAmelCase__    :      Optional[Any]       					={
    '''configuration_rwkv''': ['''RWKV_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''RwkvConfig''', '''RwkvOnnxConfig'''],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__    :      Optional[Any]       					=[
							    '''RWKV_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''RwkvForCausalLM''',
							    '''RwkvModel''',
							    '''RwkvPreTrainedModel''',
							]


if TYPE_CHECKING:
							from .configuration_rwkv import RWKV_PRETRAINED_CONFIG_ARCHIVE_MAP, RwkvConfig, RwkvOnnxConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_rwkv import (
														    RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
														    RwkvForCausalLM,
														    RwkvModel,
														    RwkvPreTrainedModel,
														)
else:
							import sys

							UpperCAmelCase__    :      int       					=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





