

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


snake_case_    						=				logging.get_logger(__name__)

snake_case_    						=				{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

snake_case_    						=				{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

snake_case_    						=				{
    '''openai-gpt''': 512,
}

def 			snake_case__    (					SCREAMING_SNAKE_CASE_	:					Optional[Any]    ):



     '''simple docstring'''





     lowercase__					:     Dict   = text.strip()
     if not text:
          return []
     lowercase__					:     Optional[int]   = text.split()
     return tokens


class    SCREAMING_SNAKE_CASE__      (__snake_case  ):





    def __init__(   self		,     a=True		,     a=None		,     a=True		,     a=None		,     a=True		,     ):
         if never_split is None:
              lowercase__					:     Union[str, Any]   = []
         lowercase__					:     Union[str, Any]   = do_lower_case
         lowercase__					:     List[str]   = set(a)
         lowercase__					:     Any   = tokenize_chinese_chars
         lowercase__					:     Dict   = strip_accents
         lowercase__					:     Union[str, Any]   = do_split_on_punc





    def    snake_case_       (   self		,     a		,     a=None):
         lowercase__					:     Tuple   = self.never_split.union(set(a)) if never_split else self.never_split
         lowercase__					:     Any   = self._clean_text(a)

         # This was added on November 1st, 2018 for the multilingual and Chinese
         # models. This is also applied to the English models now, but it doesn't
         # matter since the English models were not trained on any Chinese data
         # and generally don't have any Chinese data in them (there are Chinese
         # characters in the vocabulary because Wikipedia does have some Chinese
         # words in the English Wikipedia.).
         if self.tokenize_chinese_chars:
              lowercase__					:     Any   = self._tokenize_chinese_chars(a)
         # prevents treating the same character with different unicode codepoints as different characters
         lowercase__					:     List[Any]   = unicodedata.normalize('NFC'		,     a)
         lowercase__					:     str   = whitespace_tokenize(a)
         lowercase__					:     Optional[Any]   = []
         for token in orig_tokens:
              if token not in never_split:
                   if self.do_lower_case:
                        lowercase__					:     Tuple   = token.lower()
                        if self.strip_accents is not False:
                             lowercase__					:     Any   = self._run_strip_accents(a)
                   elif self.strip_accents:
                        lowercase__					:     Union[str, Any]   = self._run_strip_accents(a)
              split_tokens.extend(self._run_split_on_punc(a		,     a))

         lowercase__					:     int   = whitespace_tokenize(' '.join(a))
         return output_tokens





    def    snake_case_       (   self		,     a):
         lowercase__					:     Tuple   = unicodedata.normalize('NFD'		,     a)
         lowercase__					:     Union[str, Any]   = []
         for char in text:
              lowercase__					:     List[Any]   = unicodedata.category(a)
              if cat == "Mn":
                   continue
              output.append(a)
         return "".join(a)





    def    snake_case_       (   self		,     a		,     a=None):
         if not self.do_split_on_punc or (never_split is not None and text in never_split):
              return [text]
         lowercase__					:     Any   = list(a)
         lowercase__					:     Optional[int]   = 0
         lowercase__					:     Dict   = True
         lowercase__					:     Dict   = []
         while i < len(a):
              lowercase__					:     List[str]   = chars[i]
              if _is_punctuation(a):
                   output.append([char])
                   lowercase__					:     Dict   = True
              else:
                   if start_new_word:
                        output.append([])
                   lowercase__					:     Dict   = False
                   output[-1].append(a)
              i += 1

         return ["".join(a) for x in output]





    def    snake_case_       (   self		,     a):
         lowercase__					:     Union[str, Any]   = []
         for char in text:
              lowercase__					:     Optional[Any]   = ord(a)
              if self._is_chinese_char(a):
                   output.append(' ')
                   output.append(a)
                   output.append(' ')
              else:
                   output.append(a)
         return "".join(a)





    def    snake_case_       (   self		,     a):
         # This defines a "chinese character" as anything in the CJK Unicode block:
         #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
         #
         # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
         # despite its name. The modern Korean Hangul alphabet is a different block,
         # as is Japanese Hiragana and Katakana. Those alphabets are used to write
         # space-separated words, so they are not treated specially and handled
         # like the all of the other languages.
         if (
             (cp >= 0x4_E00 and cp <= 0x9_FFF)
             or (cp >= 0x3_400 and cp <= 0x4_DBF)  #
             or (cp >= 0x20_000 and cp <= 0x2A_6DF)  #
             or (cp >= 0x2A_700 and cp <= 0x2B_73F)  #
             or (cp >= 0x2B_740 and cp <= 0x2B_81F)  #
             or (cp >= 0x2B_820 and cp <= 0x2C_EAF)  #
             or (cp >= 0xF_900 and cp <= 0xF_AFF)
             or (cp >= 0x2F_800 and cp <= 0x2F_A1F)  #
         ):  #
              return True

         return False





    def    snake_case_       (   self		,     a):
         lowercase__					:     Optional[Any]   = []
         for char in text:
              lowercase__					:     Union[str, Any]   = ord(a)
              if cp == 0 or cp == 0xF_FFD or _is_control(a):
                   continue
              if _is_whitespace(a):
                   output.append(' ')
              else:
                   output.append(a)
         return "".join(a)



def 			snake_case__    (					SCREAMING_SNAKE_CASE_	:					Dict    ):



     '''simple docstring'''





     lowercase__					:     Tuple   = set()
     lowercase__					:     Optional[int]   = word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)    )
          lowercase__					:     int   = char
     return pairs

def 			snake_case__    (					SCREAMING_SNAKE_CASE_	:					str    ):



     '''simple docstring'''





     lowercase__					:     Dict   = text.replace('—'			,  '-'    )
     lowercase__					:     str   = text.replace('–'			,  '-'    )
     lowercase__					:     List[str]   = text.replace('―'			,  '-'    )
     lowercase__					:     str   = text.replace('…'			,  '...'    )
     lowercase__					:     Optional[int]   = text.replace('´'			,  '\''    )
     lowercase__					:     Union[str, Any]   = re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'			,  R' \1 '			,  SCREAMING_SNAKE_CASE_    )
     lowercase__					:     Optional[Any]   = re.sub(R'\s*\n\s*'			,  ' \n '			,  SCREAMING_SNAKE_CASE_    )
     lowercase__					:     List[Any]   = re.sub(R'[^\S\n]+'			,  ' '			,  SCREAMING_SNAKE_CASE_    )
     return text.strip()


class    SCREAMING_SNAKE_CASE__      (__snake_case  ):
    __lowerCamelCase		: Any														=  VOCAB_FILES_NAMES
    __lowerCamelCase		: Any														=  PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase		: Union[str, Any]														=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase		: Any														=  ["""input_ids""", """attention_mask"""]





    def __init__(   self		,     a		,     a		,     a="<unk>"		,     **a):
         super().__init__(unk_token=a		,     **a)

         try:
              import ftfy
              from spacy.lang.en import English

              lowercase__					:     Optional[Any]   = English()
              lowercase__					:     Any   = _nlp.tokenizer
              lowercase__					:     Any   = ftfy.fix_text
         except ImportError:
              logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.')
              lowercase__					:     Optional[Any]   = BasicTokenizer(do_lower_case=a)
              lowercase__					:     Tuple   = None

         with open(a		,     encoding='utf-8') as vocab_handle:
              lowercase__					:     Dict   = json.load(a)
         lowercase__					:     Optional[int]   = {v: k for k, v in self.encoder.items()}
         with open(a		,     encoding='utf-8') as merges_handle:
              lowercase__					:     Dict   = merges_handle.read().split('\n')[1:-1]
         lowercase__					:     Dict   = [tuple(merge.split()) for merge in merges]
         lowercase__					:     str   = dict(zip(a		,     range(len(a))))
         lowercase__					:     Tuple   = {}





    @property
    def    snake_case_       (   self):
         return True





    @property
    def    snake_case_       (   self):
         return len(self.encoder)





    def    snake_case_       (   self):
         return dict(self.encoder		,     **self.added_tokens_encoder)





    def    snake_case_       (   self		,     a):
         lowercase__					:     str   = tuple(token[:-1]) + (token[-1] + '</w>',)
         if token in self.cache:
              return self.cache[token]
         lowercase__					:     Dict   = get_pairs(a)

         if not pairs:
              return token + "</w>"

         while True:
              lowercase__					:     Dict   = min(a		,     key=lambda a: self.bpe_ranks.get(a		,     float('inf')))
              if bigram not in self.bpe_ranks:
                   break
              lowercase__   ,				lowercase__					:     str   = bigram
              lowercase__					:     Dict   = []
              lowercase__					:     Union[str, Any]   = 0
              while i < len(a):
                   try:
                        lowercase__					:     Tuple   = word.index(a		,     a)
                   except ValueError:
                        new_word.extend(word[i:])
                        break
                   else:
                        new_word.extend(word[i:j])
                        lowercase__					:     Union[str, Any]   = j

                   if word[i] == first and i < len(a) - 1 and word[i + 1] == second:
                        new_word.append(first + second)
                        i += 2
                   else:
                        new_word.append(word[i])
                        i += 1
              lowercase__					:     str   = tuple(a)
              lowercase__					:     str   = new_word
              if len(a) == 1:
                   break
              else:
                   lowercase__					:     Dict   = get_pairs(a)
         lowercase__					:     int   = ' '.join(a)
         if word == "\n  </w>":
              lowercase__					:     Tuple   = '\n</w>'
         lowercase__					:     Tuple   = word
         return word





    def    snake_case_       (   self		,     a):
         lowercase__					:     Dict   = []
         if self.fix_text is None:
              # Using BERT's BasicTokenizer
              lowercase__					:     int   = self.nlp.tokenize(a)
              for token in text:
                   split_tokens.extend(list(self.bpe(a).split(' ')))
         else:
              # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
              lowercase__					:     Union[str, Any]   = self.nlp(text_standardize(self.fix_text(a)))
              for token in text:
                   split_tokens.extend(list(self.bpe(token.text.lower()).split(' ')))
         return split_tokens





    def    snake_case_       (   self		,     a):
         return self.encoder.get(a		,     self.encoder.get(self.unk_token))





    def    snake_case_       (   self		,     a):
         return self.decoder.get(a		,     self.unk_token)





    def    snake_case_       (   self		,     a):
         lowercase__					:     str   = ''.join(a).replace('</w>'		,     ' ').strip()
         return out_string





    def    snake_case_       (   self		,     a		,     a = None):
         if not os.path.isdir(a):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
              return
         lowercase__					:     List[Any]   = os.path.join(
             a		,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
         lowercase__					:     Optional[int]   = os.path.join(
             a		,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

         with open(a		,     'w'		,     encoding='utf-8') as f:
              f.write(json.dumps(self.encoder		,     indent=2		,     sort_keys=a		,     ensure_ascii=a) + '\n')

         lowercase__					:     Optional[int]   = 0
         with open(a		,     'w'		,     encoding='utf-8') as writer:
              writer.write('#version: 0.2\n')
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,     key=lambda a: kv[1]):
                   if index != token_index:
                        logger.warning(
                            f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                            ' Please check that the tokenizer is not corrupted!')
                        lowercase__					:     int   = token_index
                   writer.write(' '.join(a) + '\n')
                   index += 1

         return vocab_file, merge_file




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


snake_case_    						=				{'''configuration_gptj''': ['''GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''GPTJConfig''', '''GPTJOnnxConfig''']}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case_    						=				[
           '''GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''GPTJForCausalLM''',
           '''GPTJForQuestionAnswering''',
           '''GPTJForSequenceClassification''',
           '''GPTJModel''',
           '''GPTJPreTrainedModel''',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case_    						=				[
           '''TFGPTJForCausalLM''',
           '''TFGPTJForQuestionAnswering''',
           '''TFGPTJForSequenceClassification''',
           '''TFGPTJModel''',
           '''TFGPTJPreTrainedModel''',
       ]

try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case_    						=				[
           '''FlaxGPTJForCausalLM''',
           '''FlaxGPTJModel''',
           '''FlaxGPTJPreTrainedModel''',
       ]


if TYPE_CHECKING:
       from .configuration_gptj import GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTJConfig, GPTJOnnxConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_gptj import (
                  GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST,
                  GPTJForCausalLM,
                  GPTJForQuestionAnswering,
                  GPTJForSequenceClassification,
                  GPTJModel,
                  GPTJPreTrainedModel,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_gptj import (
                  TFGPTJForCausalLM,
                  TFGPTJForQuestionAnswering,
                  TFGPTJForSequenceClassification,
                  TFGPTJModel,
                  TFGPTJPreTrainedModel,
              )

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel, FlaxGPTJPreTrainedModel

else:
       import sys

       snake_case_    						=				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

