code: 
  
  
  
  
  import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  SCREAMING_SNAKE_CASE__    :    Optional[Any]								=					logging.get_logger(__name__)
  
  SCREAMING_SNAKE_CASE__    :    Optional[int]								=					{
      "vocab_file": "vocab.json",
      "merges_file": "merges.txt",
  }
  
  SCREAMING_SNAKE_CASE__    :    List[str]								=					{
      "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
      "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
  }
  
  SCREAMING_SNAKE_CASE__    :    Optional[int]								=					{
      "openai-gpt": 5_12,
  }
  
  def lowercase						(     SCREAMING_SNAKE_CASE   ) ->       str:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_			=	text.strip()
  			if not text:
  						return []
  			SCREAMING_SNAKE_CASE_			=	text.split()
  			return tokens
  
  
  
  class 			a_					(    SCREAMING_SNAKE_CASE__    ):
  
  
  
  
  
  	def __init__(       self			,    SCREAMING_SNAKE_CASE=True			,    SCREAMING_SNAKE_CASE=None			,    SCREAMING_SNAKE_CASE=True			,    SCREAMING_SNAKE_CASE=None			,    SCREAMING_SNAKE_CASE=True			,    )     ->				Any:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				if never_split is None:
  							SCREAMING_SNAKE_CASE_			=	[]
  				SCREAMING_SNAKE_CASE_			=	do_lower_case
  				SCREAMING_SNAKE_CASE_			=	set(SCREAMING_SNAKE_CASE			)
  				SCREAMING_SNAKE_CASE_			=	tokenize_chinese_chars
  				SCREAMING_SNAKE_CASE_			=	strip_accents
  				SCREAMING_SNAKE_CASE_			=	do_split_on_punc
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE=None			)     ->				Dict:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				SCREAMING_SNAKE_CASE_			=	self.never_split.union(set(SCREAMING_SNAKE_CASE			)			) if never_split else self.never_split
  				SCREAMING_SNAKE_CASE_			=	self._clean_text(SCREAMING_SNAKE_CASE			)
  
  				# This was added on November 1st, 2018 for the multilingual and Chinese
  				# models. This is also applied to the English models now, but it doesn't
  				# matter since the English models were not trained on any Chinese data
  				# and generally don't have any Chinese data in them (there are Chinese
  				# characters in the vocabulary because Wikipedia does have some Chinese
  				# words in the English Wikipedia.).
  				if self.tokenize_chinese_chars:
  							SCREAMING_SNAKE_CASE_			=	self._tokenize_chinese_chars(SCREAMING_SNAKE_CASE			)
  				# prevents treating the same character with different unicode codepoints as different characters
  				SCREAMING_SNAKE_CASE_			=	unicodedata.normalize('NFC'			,    SCREAMING_SNAKE_CASE			)
  				SCREAMING_SNAKE_CASE_			=	whitespace_tokenize(SCREAMING_SNAKE_CASE			)
  				SCREAMING_SNAKE_CASE_			=	[]
  				for token in orig_tokens:
  							if token not in never_split:
  										if self.do_lower_case:
  													SCREAMING_SNAKE_CASE_			=	token.lower()
  													if self.strip_accents is not False:
  																SCREAMING_SNAKE_CASE_			=	self._run_strip_accents(SCREAMING_SNAKE_CASE			)
  										elif self.strip_accents:
  													SCREAMING_SNAKE_CASE_			=	self._run_strip_accents(SCREAMING_SNAKE_CASE			)
  							split_tokens.extend(self._run_split_on_punc(SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			)			)
  
  				SCREAMING_SNAKE_CASE_			=	whitespace_tokenize(' '.join(SCREAMING_SNAKE_CASE			)			)
  				return output_tokens
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				int:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				SCREAMING_SNAKE_CASE_			=	unicodedata.normalize('NFD'			,    SCREAMING_SNAKE_CASE			)
  				SCREAMING_SNAKE_CASE_			=	[]
  				for char in text:
  							SCREAMING_SNAKE_CASE_			=	unicodedata.category(SCREAMING_SNAKE_CASE			)
  							if cat == "Mn":
  										continue
  							output.append(SCREAMING_SNAKE_CASE			)
  				return "".join(SCREAMING_SNAKE_CASE			)
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE=None			)     ->				List[Any]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				if not self.do_split_on_punc or (never_split is not None and text in never_split):
  							return [text]
  				SCREAMING_SNAKE_CASE_			=	list(SCREAMING_SNAKE_CASE			)
  				SCREAMING_SNAKE_CASE_			=	0
  				SCREAMING_SNAKE_CASE_			=	True
  				SCREAMING_SNAKE_CASE_			=	[]
  				while i < len(SCREAMING_SNAKE_CASE			):
  							SCREAMING_SNAKE_CASE_			=	chars[i]
  							if _is_punctuation(SCREAMING_SNAKE_CASE			):
  										output.append([char]			)
  										SCREAMING_SNAKE_CASE_			=	True
  							else:
  										if start_new_word:
  													output.append([]			)
  										SCREAMING_SNAKE_CASE_			=	False
  										output[-1].append(SCREAMING_SNAKE_CASE			)
  							i += 1
  
  				return ["".join(SCREAMING_SNAKE_CASE			) for x in output]
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				List[str]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				SCREAMING_SNAKE_CASE_			=	[]
  				for char in text:
  							SCREAMING_SNAKE_CASE_			=	ord(SCREAMING_SNAKE_CASE			)
  							if self._is_chinese_char(SCREAMING_SNAKE_CASE			):
  										output.append(' '			)
  										output.append(SCREAMING_SNAKE_CASE			)
  										output.append(' '			)
  							else:
  										output.append(SCREAMING_SNAKE_CASE			)
  				return "".join(SCREAMING_SNAKE_CASE			)
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Union[str, Any]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				if (
  				    (cp >= 0X4_E_0_0 and cp <= 0X9_F_F_F)
  				    or (cp >= 0X3_4_0_0 and cp <= 0X4_D_B_F)  #
  				    or (cp >= 0X2_0_0_0_0 and cp <= 0X2_A_6_D_F)  #
  				    or (cp >= 0X2_A_7_0_0 and cp <= 0X2_B_7_3_F)  #
  				    or (cp >= 0X2_B_7_4_0 and cp <= 0X2_B_8_1_F)  #
  				    or (cp >= 0X2_B_8_2_0 and cp <= 0X2_C_E_A_F)  #
  				    or (cp >= 0XF_9_0_0 and cp <= 0XF_A_F_F)
  				    or (cp >= 0X2_F_8_0_0 and cp <= 0X2_F_A_1_F)  #
  				):  #
  							return True
  
  				return False
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Any:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				SCREAMING_SNAKE_CASE_			=	[]
  				for char in text:
  							SCREAMING_SNAKE_CASE_			=	ord(SCREAMING_SNAKE_CASE			)
  							if cp == 0 or cp == 0XF_F_F_D or _is_control(SCREAMING_SNAKE_CASE			):
  										continue
  							if _is_whitespace(SCREAMING_SNAKE_CASE			):
  										output.append(' '			)
  							else:
  										output.append(SCREAMING_SNAKE_CASE			)
  				return "".join(SCREAMING_SNAKE_CASE			)
  
  
  
  
  def lowercase						(     SCREAMING_SNAKE_CASE   ) ->       Tuple:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_			=	set()
  			SCREAMING_SNAKE_CASE_			=	word[0]
  			for char in word[1:]:
  						pairs.add((prev_char, char)   )
  						SCREAMING_SNAKE_CASE_			=	char
  			return pairs
  
  def lowercase						(     SCREAMING_SNAKE_CASE   ) ->       Optional[Any]:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_			=	text.replace('—'					,						'-'   )
  			SCREAMING_SNAKE_CASE_			=	text.replace('–'					,						'-'   )
  			SCREAMING_SNAKE_CASE_			=	text.replace('―'					,						'-'   )
  			SCREAMING_SNAKE_CASE_			=	text.replace('…'					,						'...'   )
  			SCREAMING_SNAKE_CASE_			=	text.replace('´'					,						'\''   )
  			SCREAMING_SNAKE_CASE_			=	re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'					,						r' \1 '					,						SCREAMING_SNAKE_CASE   )
  			SCREAMING_SNAKE_CASE_			=	re.sub(r'\s*\n\s*'					,						' \n '					,						SCREAMING_SNAKE_CASE   )
  			SCREAMING_SNAKE_CASE_			=	re.sub(r'[^\S\n]+'					,						' '					,						SCREAMING_SNAKE_CASE   )
  			return text.strip()
  
  
  
  class 			a_					(    SCREAMING_SNAKE_CASE__    ):
  	A						       =   VOCAB_FILES_NAMES
  	A						       =   PRETRAINED_VOCAB_FILES_MAP
  	A						       =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  	A						       =   ['''input_ids''', '''attention_mask''']
  
  
  
  
  
  	def __init__(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE="<unk>"			,    **SCREAMING_SNAKE_CASE			)     ->				Any:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				super().__init__(unk_token=SCREAMING_SNAKE_CASE			,    **SCREAMING_SNAKE_CASE			)
  
  				try:
  							import ftfy
  							from spacy.lang.en import English
  
  							SCREAMING_SNAKE_CASE_			=	English()
  							SCREAMING_SNAKE_CASE_			=	_nlp.tokenizer
  							SCREAMING_SNAKE_CASE_			=	ftfy.fix_text
  				except ImportError:
  							logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'			)
  							SCREAMING_SNAKE_CASE_			=	BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE			)
  							SCREAMING_SNAKE_CASE_			=	None
  
  				with open(SCREAMING_SNAKE_CASE			,    encoding='utf-8'			) as vocab_handle:
  							SCREAMING_SNAKE_CASE_			=	json.load(SCREAMING_SNAKE_CASE			)
  				SCREAMING_SNAKE_CASE_			=	{v: k for k, v in self.encoder.items()}
  				with open(SCREAMING_SNAKE_CASE			,    encoding='utf-8'			) as merges_handle:
  							SCREAMING_SNAKE_CASE_			=	merges_handle.read().split('\n'			)[1:-1]
  				SCREAMING_SNAKE_CASE_			=	[tuple(merge.split()			) for merge in merges]
  				SCREAMING_SNAKE_CASE_			=	dict(zip(SCREAMING_SNAKE_CASE			,    range(len(SCREAMING_SNAKE_CASE			)			)			)			)
  				SCREAMING_SNAKE_CASE_			=	{}
  
  
  
  
  
  	@property
  	def 	A_(       self			)     ->				Union[str, Any]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				return True
  
  
  
  
  
  	@property
  	def 	A_(       self			)     ->				Any:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				return len(self.encoder			)
  
  
  
  
  
  	def 	A_(       self			)     ->				Optional[Any]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				return dict(self.encoder			,    **self.added_tokens_encoder			)
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				List[Any]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				SCREAMING_SNAKE_CASE_			=	tuple(token[:-1]			) + (token[-1] + '</w>',)
  				if token in self.cache:
  							return self.cache[token]
  				SCREAMING_SNAKE_CASE_			=	get_pairs(SCREAMING_SNAKE_CASE			)
  
  				if not pairs:
  							return token + "</w>"
  
  				while True:
  							SCREAMING_SNAKE_CASE_			=	min(SCREAMING_SNAKE_CASE			,    key=lambda SCREAMING_SNAKE_CASE			: self.bpe_ranks.get(SCREAMING_SNAKE_CASE			,    float('inf'			)			)			)
  							if bigram not in self.bpe_ranks:
  										break
  							SCREAMING_SNAKE_CASE_	,		SCREAMING_SNAKE_CASE_			=	bigram
  							SCREAMING_SNAKE_CASE_			=	[]
  							SCREAMING_SNAKE_CASE_			=	0
  							while i < len(SCREAMING_SNAKE_CASE			):
  										try:
  													SCREAMING_SNAKE_CASE_			=	word.index(SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			)
  										except ValueError:
  													new_word.extend(word[i:]			)
  													break
  										else:
  													new_word.extend(word[i:j]			)
  													SCREAMING_SNAKE_CASE_			=	j
  
  										if word[i] == first and i < len(SCREAMING_SNAKE_CASE			) - 1 and word[i + 1] == second:
  													new_word.append(first + second			)
  													i += 2
  										else:
  													new_word.append(word[i]			)
  													i += 1
  							SCREAMING_SNAKE_CASE_			=	tuple(SCREAMING_SNAKE_CASE			)
  							SCREAMING_SNAKE_CASE_			=	new_word
  							if len(SCREAMING_SNAKE_CASE			) == 1:
  										break
  							else:
  										SCREAMING_SNAKE_CASE_			=	get_pairs(SCREAMING_SNAKE_CASE			)
  				SCREAMING_SNAKE_CASE_			=	' '.join(SCREAMING_SNAKE_CASE			)
  				if word == "\n  </w>":
  							SCREAMING_SNAKE_CASE_			=	'\n</w>'
  				SCREAMING_SNAKE_CASE_			=	word
  				return word
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Optional[int]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				SCREAMING_SNAKE_CASE_			=	[]
  				if self.fix_text is None:
  							# Using BERT's BasicTokenizer
  							SCREAMING_SNAKE_CASE_			=	self.nlp.tokenize(SCREAMING_SNAKE_CASE			)
  							for token in text:
  										split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE			).split(' '			)			)			)
  				else:
  							# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  							SCREAMING_SNAKE_CASE_			=	self.nlp(text_standardize(self.fix_text(SCREAMING_SNAKE_CASE			)			)			)
  							for token in text:
  										split_tokens.extend(list(self.bpe(token.text.lower()			).split(' '			)			)			)
  				return split_tokens
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Optional[int]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				return self.encoder.get(SCREAMING_SNAKE_CASE			,    self.encoder.get(self.unk_token			)			)
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Any:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				return self.decoder.get(SCREAMING_SNAKE_CASE			,    self.unk_token			)
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			)     ->				Optional[Any]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				SCREAMING_SNAKE_CASE_			=	''.join(SCREAMING_SNAKE_CASE			).replace('</w>'			,    ' '			).strip()
  				return out_string
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE = None			)     ->				Tuple[str]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				if not os.path.isdir(SCREAMING_SNAKE_CASE			):
  							logger.error(f'Vocabulary path ({save_directory}) should be a directory'			)
  							return
  				SCREAMING_SNAKE_CASE_			=	os.path.join(
  				    SCREAMING_SNAKE_CASE			,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)
  				SCREAMING_SNAKE_CASE_			=	os.path.join(
  				    SCREAMING_SNAKE_CASE			,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']			)
  
  				with open(SCREAMING_SNAKE_CASE			,    'w'			,    encoding='utf-8'			) as f:
  							f.write(json.dumps(self.encoder			,    indent=2			,    sort_keys=SCREAMING_SNAKE_CASE			,    ensure_ascii=SCREAMING_SNAKE_CASE			) + '\n'			)
  
  				SCREAMING_SNAKE_CASE_			=	0
  				with open(SCREAMING_SNAKE_CASE			,    'w'			,    encoding='utf-8'			) as writer:
  							writer.write('#version: 0.2\n'			)
  							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,    key=lambda SCREAMING_SNAKE_CASE			: kv[1]			):
  										if index != token_index:
  													logger.warning(
  													    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
  													    ' Please check that the tokenizer is not corrupted!'			)
  													SCREAMING_SNAKE_CASE_			=	token_index
  										writer.write(' '.join(SCREAMING_SNAKE_CASE			) + '\n'			)
  										index += 1
  
  				return vocab_file, merge_file
  
style_context: 
  
  
  
  
  from dataclasses import dataclass
  from typing import List, Optional, Union
  
  import numpy as np
  import PIL
  
  from ...utils import BaseOutput, is_invisible_watermark_available, is_torch_available, is_transformers_available
  
  
  
  @dataclass
  class 			a_					(    SCREAMING_SNAKE_CASE__    ):
  	A						       =   42
  
  
  if is_transformers_available() and is_torch_available() and is_invisible_watermark_available():
  				from .pipeline_stable_diffusion_xl import StableDiffusionXLPipeline
  				from .pipeline_stable_diffusion_xl_imgaimg import StableDiffusionXLImgaImgPipeline
  