code: 
  
  import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  snake_case_    						=				logging.get_logger(__name__)
  
  snake_case_    						=				{
      '''vocab_file''': '''vocab.json''',
      '''merges_file''': '''merges.txt''',
  }
  
  snake_case_    						=				{
      '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
      '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
  }
  
  snake_case_    						=				{
      '''openai-gpt''': 512,
  }
  
  
  
  def 			snake_case__    (					SCREAMING_SNAKE_CASE_	:					Optional[Any]    ):
  
  
  
       '''simple docstring'''
  
  
  
  
  
       lowercase__					:     Dict   = text.strip()
       if not text:
            return []
       lowercase__					:     Optional[int]   = text.split()
       return tokens
  
  
  class    SCREAMING_SNAKE_CASE__      (__snake_case  ):
  
  
  
  
  
      def __init__(   self		,     a=True		,     a=None		,     a=True		,     a=None		,     a=True		,     ):
           if never_split is None:
                lowercase__					:     Union[str, Any]   = []
           lowercase__					:     Union[str, Any]   = do_lower_case
           lowercase__					:     List[str]   = set(a)
           lowercase__					:     Any   = tokenize_chinese_chars
           lowercase__					:     Dict   = strip_accents
           lowercase__					:     Union[str, Any]   = do_split_on_punc
  
  
  
  
  
      def    snake_case_       (   self		,     a		,     a=None):
           lowercase__					:     Tuple   = self.never_split.union(set(a)) if never_split else self.never_split
           lowercase__					:     Any   = self._clean_text(a)
  
           # This was added on November 1st, 2018 for the multilingual and Chinese
           # models. This is also applied to the English models now, but it doesn't
           # matter since the English models were not trained on any Chinese data
           # and generally don't have any Chinese data in them (there are Chinese
           # characters in the vocabulary because Wikipedia does have some Chinese
           # words in the English Wikipedia.).
           if self.tokenize_chinese_chars:
                lowercase__					:     Any   = self._tokenize_chinese_chars(a)
           # prevents treating the same character with different unicode codepoints as different characters
           lowercase__					:     List[Any]   = unicodedata.normalize('NFC'		,     a)
           lowercase__					:     str   = whitespace_tokenize(a)
           lowercase__					:     Optional[Any]   = []
           for token in orig_tokens:
                if token not in never_split:
                     if self.do_lower_case:
                          lowercase__					:     Tuple   = token.lower()
                          if self.strip_accents is not False:
                               lowercase__					:     Any   = self._run_strip_accents(a)
                     elif self.strip_accents:
                          lowercase__					:     Union[str, Any]   = self._run_strip_accents(a)
                split_tokens.extend(self._run_split_on_punc(a		,     a))
  
           lowercase__					:     int   = whitespace_tokenize(' '.join(a))
           return output_tokens
  
  
  
  
  
      def    snake_case_       (   self		,     a):
           lowercase__					:     Tuple   = unicodedata.normalize('NFD'		,     a)
           lowercase__					:     Union[str, Any]   = []
           for char in text:
                lowercase__					:     List[Any]   = unicodedata.category(a)
                if cat == "Mn":
                     continue
                output.append(a)
           return "".join(a)
  
  
  
  
  
      def    snake_case_       (   self		,     a		,     a=None):
           if not self.do_split_on_punc or (never_split is not None and text in never_split):
                return [text]
           lowercase__					:     Any   = list(a)
           lowercase__					:     Optional[int]   = 0
           lowercase__					:     Dict   = True
           lowercase__					:     Dict   = []
           while i < len(a):
                lowercase__					:     List[str]   = chars[i]
                if _is_punctuation(a):
                     output.append([char])
                     lowercase__					:     Dict   = True
                else:
                     if start_new_word:
                          output.append([])
                     lowercase__					:     Dict   = False
                     output[-1].append(a)
                i += 1
  
           return ["".join(a) for x in output]
  
  
  
  
  
      def    snake_case_       (   self		,     a):
           lowercase__					:     Union[str, Any]   = []
           for char in text:
                lowercase__					:     Optional[Any]   = ord(a)
                if self._is_chinese_char(a):
                     output.append(' ')
                     output.append(a)
                     output.append(' ')
                else:
                     output.append(a)
           return "".join(a)
  
  
  
  
  
      def    snake_case_       (   self		,     a):
           # This defines a "chinese character" as anything in the CJK Unicode block:
           #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
           #
           # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
           # despite its name. The modern Korean Hangul alphabet is a different block,
           # as is Japanese Hiragana and Katakana. Those alphabets are used to write
           # space-separated words, so they are not treated specially and handled
           # like the all of the other languages.
           if (
               (cp >= 0x4_E00 and cp <= 0x9_FFF)
               or (cp >= 0x3_400 and cp <= 0x4_DBF)  #
               or (cp >= 0x20_000 and cp <= 0x2A_6DF)  #
               or (cp >= 0x2A_700 and cp <= 0x2B_73F)  #
               or (cp >= 0x2B_740 and cp <= 0x2B_81F)  #
               or (cp >= 0x2B_820 and cp <= 0x2C_EAF)  #
               or (cp >= 0xF_900 and cp <= 0xF_AFF)
               or (cp >= 0x2F_800 and cp <= 0x2F_A1F)  #
           ):  #
                return True
  
           return False
  
  
  
  
  
      def    snake_case_       (   self		,     a):
           lowercase__					:     Optional[Any]   = []
           for char in text:
                lowercase__					:     Union[str, Any]   = ord(a)
                if cp == 0 or cp == 0xF_FFD or _is_control(a):
                     continue
                if _is_whitespace(a):
                     output.append(' ')
                else:
                     output.append(a)
           return "".join(a)
  
  
  
  def 			snake_case__    (					SCREAMING_SNAKE_CASE_	:					Dict    ):
  
  
  
       '''simple docstring'''
  
  
  
  
  
       lowercase__					:     Tuple   = set()
       lowercase__					:     Optional[int]   = word[0]
       for char in word[1:]:
            pairs.add((prev_char, char)    )
            lowercase__					:     int   = char
       return pairs
  
  
  
  def 			snake_case__    (					SCREAMING_SNAKE_CASE_	:					str    ):
  
  
  
       '''simple docstring'''
  
  
  
  
  
       lowercase__					:     Dict   = text.replace('—'			,  '-'    )
       lowercase__					:     str   = text.replace('–'			,  '-'    )
       lowercase__					:     List[str]   = text.replace('―'			,  '-'    )
       lowercase__					:     str   = text.replace('…'			,  '...'    )
       lowercase__					:     Optional[int]   = text.replace('´'			,  '\''    )
       lowercase__					:     Union[str, Any]   = re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'			,  R' \1 '			,  SCREAMING_SNAKE_CASE_    )
       lowercase__					:     Optional[Any]   = re.sub(R'\s*\n\s*'			,  ' \n '			,  SCREAMING_SNAKE_CASE_    )
       lowercase__					:     List[Any]   = re.sub(R'[^\S\n]+'			,  ' '			,  SCREAMING_SNAKE_CASE_    )
       return text.strip()
  
  
  class    SCREAMING_SNAKE_CASE__      (__snake_case  ):
      __lowerCamelCase		: Any														=  VOCAB_FILES_NAMES
      __lowerCamelCase		: Any														=  PRETRAINED_VOCAB_FILES_MAP
      __lowerCamelCase		: Union[str, Any]														=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __lowerCamelCase		: Any														=  ["""input_ids""", """attention_mask"""]
  
  
  
  
  
      def __init__(   self		,     a		,     a		,     a="<unk>"		,     **a):
           super().__init__(unk_token=a		,     **a)
  
           try:
                import ftfy
                from spacy.lang.en import English
  
                lowercase__					:     Optional[Any]   = English()
                lowercase__					:     Any   = _nlp.tokenizer
                lowercase__					:     Any   = ftfy.fix_text
           except ImportError:
                logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.')
                lowercase__					:     Optional[Any]   = BasicTokenizer(do_lower_case=a)
                lowercase__					:     Tuple   = None
  
           with open(a		,     encoding='utf-8') as vocab_handle:
                lowercase__					:     Dict   = json.load(a)
           lowercase__					:     Optional[int]   = {v: k for k, v in self.encoder.items()}
           with open(a		,     encoding='utf-8') as merges_handle:
                lowercase__					:     Dict   = merges_handle.read().split('\n')[1:-1]
           lowercase__					:     Dict   = [tuple(merge.split()) for merge in merges]
           lowercase__					:     str   = dict(zip(a		,     range(len(a))))
           lowercase__					:     Tuple   = {}
  
  
  
  
  
      @property
      def    snake_case_       (   self):
           return True
  
  
  
  
  
      @property
      def    snake_case_       (   self):
           return len(self.encoder)
  
  
  
  
  
      def    snake_case_       (   self):
           return dict(self.encoder		,     **self.added_tokens_encoder)
  
  
  
  
  
      def    snake_case_       (   self		,     a):
           lowercase__					:     str   = tuple(token[:-1]) + (token[-1] + '</w>',)
           if token in self.cache:
                return self.cache[token]
           lowercase__					:     Dict   = get_pairs(a)
  
           if not pairs:
                return token + "</w>"
  
           while True:
                lowercase__					:     Dict   = min(a		,     key=lambda a: self.bpe_ranks.get(a		,     float('inf')))
                if bigram not in self.bpe_ranks:
                     break
                lowercase__					:     str   = bigram
                lowercase__					:     Dict   = []
                lowercase__					:     Union[str, Any]   = 0
                while i < len(a):
                     try:
                          lowercase__					:     Tuple   = word.index(a		,     a)
                     except ValueError:
                          new_word.extend(word[i:])
                          break
                     else:
                          new_word.extend(word[i:j])
                          lowercase__					:     Union[str, Any]   = j
  
                     if word[i] == first and i < len(a) - 1 and word[i + 1] == second:
                          new_word.append(first + second)
                          i += 2
                     else:
                          new_word.append(word[i])
                          i += 1
                lowercase__					:     str   = tuple(a)
                lowercase__					:     str   = new_word
                if len(a) == 1:
                     break
                else:
                     lowercase__					:     Dict   = get_pairs(a)
           lowercase__					:     int   = ' '.join(a)
           if word == "\n  </w>":
                lowercase__					:     Tuple   = '\n</w>'
           lowercase__					:     Tuple   = word
           return word
  
  
  
  
  
      def    snake_case_       (   self		,     a):
           lowercase__					:     Dict   = []
           if self.fix_text is None:
                # Using BERT's BasicTokenizer
                lowercase__					:     int   = self.nlp.tokenize(a)
                for token in text:
                     split_tokens.extend(list(self.bpe(a).split(' ')))
           else:
                # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                lowercase__					:     Union[str, Any]   = self.nlp(text_standardize(self.fix_text(a)))
                for token in text:
                     split_tokens.extend(list(self.bpe(token.text.lower()).split(' ')))
           return split_tokens
  
  
  
  
  
      def    snake_case_       (   self		,     a):
           return self.encoder.get(a		,     self.encoder.get(self.unk_token))
  
  
  
  
  
      def    snake_case_       (   self		,     a):
           return self.decoder.get(a		,     self.unk_token)
  
  
  
  
  
      def    snake_case_       (   self		,     a):
           lowercase__					:     str   = ''.join(a).replace('</w>'		,     ' ').strip()
           return out_string
  
  
  
  
  
      def    snake_case_       (   self		,     a		,     a = None):
           if not os.path.isdir(a):
                logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
                return
           lowercase__					:     List[Any]   = os.path.join(
               a		,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
           lowercase__					:     Optional[int]   = os.path.join(
               a		,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])
  
           with open(a		,     'w'		,     encoding='utf-8') as f:
                f.write(json.dumps(self.encoder		,     indent=2		,     sort_keys=a		,     ensure_ascii=a) + '\n')
  
           lowercase__					:     Optional[int]   = 0
           with open(a		,     'w'		,     encoding='utf-8') as writer:
                writer.write('#version: 0.2\n')
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,     key=lambda a: kv[1]):
                     if index != token_index:
                          logger.warning(
                              f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                              ' Please check that the tokenizer is not corrupted!')
                          lowercase__					:     int   = token_index
                     writer.write(' '.join(a) + '\n')
                     index += 1
  
           return vocab_file, merge_file
  
  
style_context: 
  
  from dataclasses import dataclass
  from typing import List, Optional, Union
  
  import numpy as np
  import PIL
  
  from ...utils import BaseOutput, is_invisible_watermark_available, is_torch_available, is_transformers_available
  
  
  @dataclass
  class    SCREAMING_SNAKE_CASE__      (__snake_case  ):
      __lowerCamelCase		: Union[List[PIL.Image.Image], np.ndarray]
  
  
  if is_transformers_available() and is_torch_available() and is_invisible_watermark_available():
         from .pipeline_stable_diffusion_xl import StableDiffusionXLPipeline
         from .pipeline_stable_diffusion_xl_imgaimg import StableDiffusionXLImgaImgPipeline
  
  