




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


SCREAMING_SNAKE_CASE__    :    Optional[Any]								=					logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__    :    Optional[int]								=					{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

SCREAMING_SNAKE_CASE__    :    List[str]								=					{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

SCREAMING_SNAKE_CASE__    :    Optional[int]								=					{
    "openai-gpt": 5_12,
}

def lowercase						(     SCREAMING_SNAKE_CASE   ) ->       str:


			'''simple docstring'''







			SCREAMING_SNAKE_CASE_			=	text.strip()
			if not text:
						return []
			SCREAMING_SNAKE_CASE_			=	text.split()
			return tokens



class 			a_					(    SCREAMING_SNAKE_CASE__    ):





	def __init__(       self			,    SCREAMING_SNAKE_CASE=True			,    SCREAMING_SNAKE_CASE=None			,    SCREAMING_SNAKE_CASE=True			,    SCREAMING_SNAKE_CASE=None			,    SCREAMING_SNAKE_CASE=True			,    )     ->				Any:




				"""simple docstring"""



				if never_split is None:
							SCREAMING_SNAKE_CASE_			=	[]
				SCREAMING_SNAKE_CASE_			=	do_lower_case
				SCREAMING_SNAKE_CASE_			=	set(SCREAMING_SNAKE_CASE			)
				SCREAMING_SNAKE_CASE_			=	tokenize_chinese_chars
				SCREAMING_SNAKE_CASE_			=	strip_accents
				SCREAMING_SNAKE_CASE_			=	do_split_on_punc





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE=None			)     ->				Dict:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			=	self.never_split.union(set(SCREAMING_SNAKE_CASE			)			) if never_split else self.never_split
				SCREAMING_SNAKE_CASE_			=	self._clean_text(SCREAMING_SNAKE_CASE			)

				# This was added on November 1st, 2018 for the multilingual and Chinese
				# models. This is also applied to the English models now, but it doesn't
				# matter since the English models were not trained on any Chinese data
				# and generally don't have any Chinese data in them (there are Chinese
				# characters in the vocabulary because Wikipedia does have some Chinese
				# words in the English Wikipedia.).
				if self.tokenize_chinese_chars:
							SCREAMING_SNAKE_CASE_			=	self._tokenize_chinese_chars(SCREAMING_SNAKE_CASE			)
				# prevents treating the same character with different unicode codepoints as different characters
				SCREAMING_SNAKE_CASE_			=	unicodedata.normalize('NFC'			,    SCREAMING_SNAKE_CASE			)
				SCREAMING_SNAKE_CASE_			=	whitespace_tokenize(SCREAMING_SNAKE_CASE			)
				SCREAMING_SNAKE_CASE_			=	[]
				for token in orig_tokens:
							if token not in never_split:
										if self.do_lower_case:
													SCREAMING_SNAKE_CASE_			=	token.lower()
													if self.strip_accents is not False:
																SCREAMING_SNAKE_CASE_			=	self._run_strip_accents(SCREAMING_SNAKE_CASE			)
										elif self.strip_accents:
													SCREAMING_SNAKE_CASE_			=	self._run_strip_accents(SCREAMING_SNAKE_CASE			)
							split_tokens.extend(self._run_split_on_punc(SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			)			)

				SCREAMING_SNAKE_CASE_			=	whitespace_tokenize(' '.join(SCREAMING_SNAKE_CASE			)			)
				return output_tokens





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				int:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			=	unicodedata.normalize('NFD'			,    SCREAMING_SNAKE_CASE			)
				SCREAMING_SNAKE_CASE_			=	[]
				for char in text:
							SCREAMING_SNAKE_CASE_			=	unicodedata.category(SCREAMING_SNAKE_CASE			)
							if cat == "Mn":
										continue
							output.append(SCREAMING_SNAKE_CASE			)
				return "".join(SCREAMING_SNAKE_CASE			)





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE=None			)     ->				List[Any]:




				"""simple docstring"""



				if not self.do_split_on_punc or (never_split is not None and text in never_split):
							return [text]
				SCREAMING_SNAKE_CASE_			=	list(SCREAMING_SNAKE_CASE			)
				SCREAMING_SNAKE_CASE_			=	0
				SCREAMING_SNAKE_CASE_			=	True
				SCREAMING_SNAKE_CASE_			=	[]
				while i < len(SCREAMING_SNAKE_CASE			):
							SCREAMING_SNAKE_CASE_			=	chars[i]
							if _is_punctuation(SCREAMING_SNAKE_CASE			):
										output.append([char]			)
										SCREAMING_SNAKE_CASE_			=	True
							else:
										if start_new_word:
													output.append([]			)
										SCREAMING_SNAKE_CASE_			=	False
										output[-1].append(SCREAMING_SNAKE_CASE			)
							i += 1

				return ["".join(SCREAMING_SNAKE_CASE			) for x in output]





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				List[str]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			=	[]
				for char in text:
							SCREAMING_SNAKE_CASE_			=	ord(SCREAMING_SNAKE_CASE			)
							if self._is_chinese_char(SCREAMING_SNAKE_CASE			):
										output.append(' '			)
										output.append(SCREAMING_SNAKE_CASE			)
										output.append(' '			)
							else:
										output.append(SCREAMING_SNAKE_CASE			)
				return "".join(SCREAMING_SNAKE_CASE			)





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Union[str, Any]:




				"""simple docstring"""



				if (
				    (cp >= 0X4_E_0_0 and cp <= 0X9_F_F_F)
				    or (cp >= 0X3_4_0_0 and cp <= 0X4_D_B_F)  #
				    or (cp >= 0X2_0_0_0_0 and cp <= 0X2_A_6_D_F)  #
				    or (cp >= 0X2_A_7_0_0 and cp <= 0X2_B_7_3_F)  #
				    or (cp >= 0X2_B_7_4_0 and cp <= 0X2_B_8_1_F)  #
				    or (cp >= 0X2_B_8_2_0 and cp <= 0X2_C_E_A_F)  #
				    or (cp >= 0XF_9_0_0 and cp <= 0XF_A_F_F)
				    or (cp >= 0X2_F_8_0_0 and cp <= 0X2_F_A_1_F)  #
				):  #
							return True

				return False





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Any:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			=	[]
				for char in text:
							SCREAMING_SNAKE_CASE_			=	ord(SCREAMING_SNAKE_CASE			)
							if cp == 0 or cp == 0XF_F_F_D or _is_control(SCREAMING_SNAKE_CASE			):
										continue
							if _is_whitespace(SCREAMING_SNAKE_CASE			):
										output.append(' '			)
							else:
										output.append(SCREAMING_SNAKE_CASE			)
				return "".join(SCREAMING_SNAKE_CASE			)




def lowercase						(     SCREAMING_SNAKE_CASE   ) ->       Tuple:


			'''simple docstring'''







			SCREAMING_SNAKE_CASE_			=	set()
			SCREAMING_SNAKE_CASE_			=	word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)   )
						SCREAMING_SNAKE_CASE_			=	char
			return pairs

def lowercase						(     SCREAMING_SNAKE_CASE   ) ->       Optional[Any]:


			'''simple docstring'''







			SCREAMING_SNAKE_CASE_			=	text.replace('—'					,						'-'   )
			SCREAMING_SNAKE_CASE_			=	text.replace('–'					,						'-'   )
			SCREAMING_SNAKE_CASE_			=	text.replace('―'					,						'-'   )
			SCREAMING_SNAKE_CASE_			=	text.replace('…'					,						'...'   )
			SCREAMING_SNAKE_CASE_			=	text.replace('´'					,						'\''   )
			SCREAMING_SNAKE_CASE_			=	re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'					,						r' \1 '					,						SCREAMING_SNAKE_CASE   )
			SCREAMING_SNAKE_CASE_			=	re.sub(r'\s*\n\s*'					,						' \n '					,						SCREAMING_SNAKE_CASE   )
			SCREAMING_SNAKE_CASE_			=	re.sub(r'[^\S\n]+'					,						' '					,						SCREAMING_SNAKE_CASE   )
			return text.strip()



class 			a_					(    SCREAMING_SNAKE_CASE__    ):
	A						       =   VOCAB_FILES_NAMES
	A						       =   PRETRAINED_VOCAB_FILES_MAP
	A						       =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A						       =   ['''input_ids''', '''attention_mask''']





	def __init__(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE="<unk>"			,    **SCREAMING_SNAKE_CASE			)     ->				Any:




				"""simple docstring"""



				super().__init__(unk_token=SCREAMING_SNAKE_CASE			,    **SCREAMING_SNAKE_CASE			)

				try:
							import ftfy
							from spacy.lang.en import English

							SCREAMING_SNAKE_CASE_			=	English()
							SCREAMING_SNAKE_CASE_			=	_nlp.tokenizer
							SCREAMING_SNAKE_CASE_			=	ftfy.fix_text
				except ImportError:
							logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'			)
							SCREAMING_SNAKE_CASE_			=	BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE			)
							SCREAMING_SNAKE_CASE_			=	None

				with open(SCREAMING_SNAKE_CASE			,    encoding='utf-8'			) as vocab_handle:
							SCREAMING_SNAKE_CASE_			=	json.load(SCREAMING_SNAKE_CASE			)
				SCREAMING_SNAKE_CASE_			=	{v: k for k, v in self.encoder.items()}
				with open(SCREAMING_SNAKE_CASE			,    encoding='utf-8'			) as merges_handle:
							SCREAMING_SNAKE_CASE_			=	merges_handle.read().split('\n'			)[1:-1]
				SCREAMING_SNAKE_CASE_			=	[tuple(merge.split()			) for merge in merges]
				SCREAMING_SNAKE_CASE_			=	dict(zip(SCREAMING_SNAKE_CASE			,    range(len(SCREAMING_SNAKE_CASE			)			)			)			)
				SCREAMING_SNAKE_CASE_			=	{}





	@property
	def 	A_(       self			)     ->				Union[str, Any]:




				"""simple docstring"""



				return True





	@property
	def 	A_(       self			)     ->				Any:




				"""simple docstring"""



				return len(self.encoder			)





	def 	A_(       self			)     ->				Optional[Any]:




				"""simple docstring"""



				return dict(self.encoder			,    **self.added_tokens_encoder			)





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				List[Any]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			=	tuple(token[:-1]			) + (token[-1] + '</w>',)
				if token in self.cache:
							return self.cache[token]
				SCREAMING_SNAKE_CASE_			=	get_pairs(SCREAMING_SNAKE_CASE			)

				if not pairs:
							return token + "</w>"

				while True:
							SCREAMING_SNAKE_CASE_			=	min(SCREAMING_SNAKE_CASE			,    key=lambda SCREAMING_SNAKE_CASE			: self.bpe_ranks.get(SCREAMING_SNAKE_CASE			,    float('inf'			)			)			)
							if bigram not in self.bpe_ranks:
										break
							SCREAMING_SNAKE_CASE_	,		SCREAMING_SNAKE_CASE_			=	bigram
							SCREAMING_SNAKE_CASE_			=	[]
							SCREAMING_SNAKE_CASE_			=	0
							while i < len(SCREAMING_SNAKE_CASE			):
										try:
													SCREAMING_SNAKE_CASE_			=	word.index(SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			)
										except ValueError:
													new_word.extend(word[i:]			)
													break
										else:
													new_word.extend(word[i:j]			)
													SCREAMING_SNAKE_CASE_			=	j

										if word[i] == first and i < len(SCREAMING_SNAKE_CASE			) - 1 and word[i + 1] == second:
													new_word.append(first + second			)
													i += 2
										else:
													new_word.append(word[i]			)
													i += 1
							SCREAMING_SNAKE_CASE_			=	tuple(SCREAMING_SNAKE_CASE			)
							SCREAMING_SNAKE_CASE_			=	new_word
							if len(SCREAMING_SNAKE_CASE			) == 1:
										break
							else:
										SCREAMING_SNAKE_CASE_			=	get_pairs(SCREAMING_SNAKE_CASE			)
				SCREAMING_SNAKE_CASE_			=	' '.join(SCREAMING_SNAKE_CASE			)
				if word == "\n  </w>":
							SCREAMING_SNAKE_CASE_			=	'\n</w>'
				SCREAMING_SNAKE_CASE_			=	word
				return word





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Optional[int]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			=	[]
				if self.fix_text is None:
							# Using BERT's BasicTokenizer
							SCREAMING_SNAKE_CASE_			=	self.nlp.tokenize(SCREAMING_SNAKE_CASE			)
							for token in text:
										split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE			).split(' '			)			)			)
				else:
							# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
							SCREAMING_SNAKE_CASE_			=	self.nlp(text_standardize(self.fix_text(SCREAMING_SNAKE_CASE			)			)			)
							for token in text:
										split_tokens.extend(list(self.bpe(token.text.lower()			).split(' '			)			)			)
				return split_tokens





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Optional[int]:




				"""simple docstring"""



				return self.encoder.get(SCREAMING_SNAKE_CASE			,    self.encoder.get(self.unk_token			)			)





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Any:




				"""simple docstring"""



				return self.decoder.get(SCREAMING_SNAKE_CASE			,    self.unk_token			)





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Optional[Any]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			=	''.join(SCREAMING_SNAKE_CASE			).replace('</w>'			,    ' '			).strip()
				return out_string





	def 	A_(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE = None			)     ->				Tuple[str]:




				"""simple docstring"""



				if not os.path.isdir(SCREAMING_SNAKE_CASE			):
							logger.error(f'Vocabulary path ({save_directory}) should be a directory'			)
							return
				SCREAMING_SNAKE_CASE_			=	os.path.join(
				    SCREAMING_SNAKE_CASE			,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)
				SCREAMING_SNAKE_CASE_			=	os.path.join(
				    SCREAMING_SNAKE_CASE			,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']			)

				with open(SCREAMING_SNAKE_CASE			,    'w'			,    encoding='utf-8'			) as f:
							f.write(json.dumps(self.encoder			,    indent=2			,    sort_keys=SCREAMING_SNAKE_CASE			,    ensure_ascii=SCREAMING_SNAKE_CASE			) + '\n'			)

				SCREAMING_SNAKE_CASE_			=	0
				with open(SCREAMING_SNAKE_CASE			,    'w'			,    encoding='utf-8'			) as writer:
							writer.write('#version: 0.2\n'			)
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,    key=lambda SCREAMING_SNAKE_CASE			: kv[1]			):
										if index != token_index:
													logger.warning(
													    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
													    ' Please check that the tokenizer is not corrupted!'			)
													SCREAMING_SNAKE_CASE_			=	token_index
										writer.write(' '.join(SCREAMING_SNAKE_CASE			) + '\n'			)
										index += 1

				return vocab_file, merge_file






from dataclasses import dataclass
from typing import List, Optional, Union

import numpy as np
import PIL

from ...utils import BaseOutput, is_invisible_watermark_available, is_torch_available, is_transformers_available



@dataclass
class 			a_					(    SCREAMING_SNAKE_CASE__    ):
	A						       =   42


if is_transformers_available() and is_torch_available() and is_invisible_watermark_available():
				from .pipeline_stable_diffusion_xl import StableDiffusionXLPipeline
				from .pipeline_stable_diffusion_xl_imgaimg import StableDiffusionXLImgaImgPipeline
