

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_UpperCAmelCase    	=			logging.get_logger(__name__)

_UpperCAmelCase    	=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_UpperCAmelCase    	=			{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

_UpperCAmelCase    	=			{
    'openai-gpt': 512,
}


def 			__UpperCamelCase    (lowerCAmelCase					:  Any     )     ->  List[str]:
      A    =				text.strip()
      if not text:
            return []
      A    =				text.split()
      return tokens



class        _UpperCAmelCase (   __lowercase			):
  '''simple docstring'''







  def __init__( self	: List[str]		,						UpperCamelCase__	: List[str]=True		,						UpperCamelCase__	: List[Any]=None		,						UpperCamelCase__	: Any=True		,						UpperCamelCase__	: Any=None		,						UpperCamelCase__	: str=True		,						):
        if never_split is None:
              A    =				[]
        A    =				do_lower_case
        A    =				set(A_ )
        A    =				tokenize_chinese_chars
        A    =				strip_accents
        A    =				do_split_on_punc







  def 	UpperCamelCase       ( self	: Union[str, Any]		,						UpperCamelCase__	: Union[str, Any]		,						UpperCamelCase__	: int=None ):
        A    =				self.never_split.union(set(A_ ) ) if never_split else self.never_split
        A    =				self._clean_text(A_ )

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
              A    =				self._tokenize_chinese_chars(A_ )
        # prevents treating the same character with different unicode codepoints as different characters
        A    =				unicodedata.normalize('NFC'		,						A_ )
        A    =				whitespace_tokenize(A_ )
        A    =				[]
        for token in orig_tokens:
              if token not in never_split:
                    if self.do_lower_case:
                          A    =				token.lower()
                          if self.strip_accents is not False:
                                A    =				self._run_strip_accents(A_ )
                    elif self.strip_accents:
                          A    =				self._run_strip_accents(A_ )
              split_tokens.extend(self._run_split_on_punc(A_		,						A_ ) )

        A    =				whitespace_tokenize(' '.join(A_ ) )
        return output_tokens







  def 	UpperCamelCase       ( self	: Optional[int]		,						UpperCamelCase__	: Union[str, Any] ):
        A    =				unicodedata.normalize('NFD'		,						A_ )
        A    =				[]
        for char in text:
              A    =				unicodedata.category(A_ )
              if cat == "Mn":
                    continue
              output.append(A_ )
        return "".join(A_ )







  def 	UpperCamelCase       ( self	: Dict		,						UpperCamelCase__	: int		,						UpperCamelCase__	: Union[str, Any]=None ):
        if not self.do_split_on_punc or (never_split is not None and text in never_split):
              return [text]
        A    =				list(A_ )
        A    =				0
        A    =				True
        A    =				[]
        while i < len(A_ ):
              A    =				chars[i]
              if _is_punctuation(A_ ):
                    output.append([char] )
                    A    =				True
              else:
                    if start_new_word:
                          output.append([] )
                    A    =				False
                    output[-1].append(A_ )
              i += 1

        return ["".join(A_ ) for x in output]







  def 	UpperCamelCase       ( self	: List[str]		,						UpperCamelCase__	: Optional[int] ):
        A    =				[]
        for char in text:
              A    =				ord(A_ )
              if self._is_chinese_char(A_ ):
                    output.append(' ' )
                    output.append(A_ )
                    output.append(' ' )
              else:
                    output.append(A_ )
        return "".join(A_ )







  def 	UpperCamelCase       ( self	: List[Any]		,						UpperCamelCase__	: Optional[int] ):
        # This defines a "chinese character" as anything in the CJK Unicode block:
        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
        #
        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
        # despite its name. The modern Korean Hangul alphabet is a different block,
        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
        # space-separated words, so they are not treated specially and handled
        # like the all of the other languages.
        if (
            (cp >= 0X4_e00 and cp <= 0X9_fff)
            or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
            or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
            or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
            or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
            or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
            or (cp >= 0Xf_900 and cp <= 0Xf_aff)
            or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
        ):  #
              return True

        return False







  def 	UpperCamelCase       ( self	: Optional[Any]		,						UpperCamelCase__	: Tuple ):
        A    =				[]
        for char in text:
              A    =				ord(A_ )
              if cp == 0 or cp == 0Xf_ffd or _is_control(A_ ):
                    continue
              if _is_whitespace(A_ ):
                    output.append(' ' )
              else:
                    output.append(A_ )
        return "".join(A_ )


def 			__UpperCamelCase    (lowerCAmelCase					:  List[str]     )     ->  int:
      A    =				set()
      A    =				word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)     )
            A    =				char
      return pairs


def 			__UpperCamelCase    (lowerCAmelCase					:  Optional[Any]     )     ->  List[str]:
      A    =				text.replace('—',    '-'     )
      A    =				text.replace('–',    '-'     )
      A    =				text.replace('―',    '-'     )
      A    =				text.replace('…',    '...'     )
      A    =				text.replace('´',    '\''     )
      A    =				re.sub(r'(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)',    r' \1 ',    lowercase__     )
      A    =				re.sub(r'\s*\n\s*',    ' \n ',    lowercase__     )
      A    =				re.sub(r'[^\S\n]+',    ' ',    lowercase__     )
      return text.strip()







class        _UpperCAmelCase (   __lowercase			):
  '''simple docstring'''







  SCREAMING_SNAKE_CASE							:				Optional[Any]					      =				VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE							:				int					      =				PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE							:				Dict					      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE							:				List[str]					      =				['''input_ids''', '''attention_mask''']







  def __init__( self	: Union[str, Any]		,						UpperCamelCase__	: int		,						UpperCamelCase__	: Dict		,						UpperCamelCase__	: Dict="<unk>"		,						**UpperCamelCase__	: Union[str, Any] ):
        super().__init__(unk_token=A_		,						**A_ )

        try:
              import ftfy
              from spacy.lang.en import English

              A    =				English()
              A    =				_nlp.tokenizer
              A    =				ftfy.fix_text
        except ImportError:
              logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.' )
              A    =				BasicTokenizer(do_lower_case=A_ )
              A    =				None

        with open(A_		,						encoding='utf-8' ) as vocab_handle:
              A    =				json.load(A_ )
        A    =				{v: k for k, v in self.encoder.items()}
        with open(A_		,						encoding='utf-8' ) as merges_handle:
              A    =				merges_handle.read().split('\n' )[1:-1]
        A    =				[tuple(merge.split() ) for merge in merges]
        A    =				dict(zip(A_		,						range(len(A_ ) ) ) )
        A    =				{}







  @property
  def 	UpperCamelCase       ( self	: List[str] ):
        return True







  @property
  def 	UpperCamelCase       ( self	: Optional[Any] ):
        return len(self.encoder )







  def 	UpperCamelCase       ( self	: Optional[int] ):
        return dict(self.encoder		,						**self.added_tokens_encoder )







  def 	UpperCamelCase       ( self	: Union[str, Any]		,						UpperCamelCase__	: Tuple ):
        A    =				tuple(token[:-1] ) + (token[-1] + '</w>',)
        if token in self.cache:
              return self.cache[token]
        A    =				get_pairs(A_ )

        if not pairs:
              return token + "</w>"

        while True:
              A    =				min(A_		,						key=lambda UpperCamelCase__ : self.bpe_ranks.get(A_		,						float('inf' ) ) )
              if bigram not in self.bpe_ranks:
                    break
              A ,	A    =				bigram
              A    =				[]
              A    =				0
              while i < len(A_ ):
                    try:
                          A    =				word.index(A_		,						A_ )
                    except ValueError:
                          new_word.extend(word[i:] )
                          break
                    else:
                          new_word.extend(word[i:j] )
                          A    =				j

                    if word[i] == first and i < len(A_ ) - 1 and word[i + 1] == second:
                          new_word.append(first + second )
                          i += 2
                    else:
                          new_word.append(word[i] )
                          i += 1
              A    =				tuple(A_ )
              A    =				new_word
              if len(A_ ) == 1:
                    break
              else:
                    A    =				get_pairs(A_ )
        A    =				' '.join(A_ )
        if word == "\n  </w>":
              A    =				'\n</w>'
        A    =				word
        return word







  def 	UpperCamelCase       ( self	: Any		,						UpperCamelCase__	: Any ):
        A    =				[]
        if self.fix_text is None:
              # Using BERT's BasicTokenizer
              A    =				self.nlp.tokenize(A_ )
              for token in text:
                    split_tokens.extend(list(self.bpe(A_ ).split(' ' ) ) )
        else:
              # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
              A    =				self.nlp(text_standardize(self.fix_text(A_ ) ) )
              for token in text:
                    split_tokens.extend(list(self.bpe(token.text.lower() ).split(' ' ) ) )
        return split_tokens







  def 	UpperCamelCase       ( self	: List[Any]		,						UpperCamelCase__	: List[str] ):
        return self.encoder.get(A_		,						self.encoder.get(self.unk_token ) )







  def 	UpperCamelCase       ( self	: Dict		,						UpperCamelCase__	: Union[str, Any] ):
        return self.decoder.get(A_		,						self.unk_token )







  def 	UpperCamelCase       ( self	: List[Any]		,						UpperCamelCase__	: Any ):
        A    =				''.join(A_ ).replace('</w>'		,						' ' ).strip()
        return out_string







  def 	UpperCamelCase       ( self	: Union[str, Any]		,						UpperCamelCase__	: str		,						UpperCamelCase__	: Optional[str] = None ):
        if not os.path.isdir(A_ ):
              logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
              return
        A    =				os.path.join(
            A_		,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
        A    =				os.path.join(
            A_		,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'] )

        with open(A_		,						'w'		,						encoding='utf-8' ) as f:
              f.write(json.dumps(self.encoder		,						indent=2		,						sort_keys=A_		,						ensure_ascii=A_ ) + '\n' )

        A    =				0
        with open(A_		,						'w'		,						encoding='utf-8' ) as writer:
              writer.write('#version: 0.2\n' )
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,						key=lambda UpperCamelCase__ : kv[1] ):
                    if index != token_index:
                          logger.warning(
                              f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                              ' Please check that the tokenizer is not corrupted!' )
                          A    =				token_index
                    writer.write(' '.join(A_ ) + '\n' )
                    index += 1

        return vocab_file, merge_file






from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


_UpperCAmelCase    	=			logging.get_logger(__name__)

_UpperCAmelCase    	=			{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_UpperCAmelCase    	=			{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
    "tokenizer_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/tokenizer.json"},
}

_UpperCAmelCase    	=			{
    "openai-gpt": 512,
}



class        _UpperCAmelCase (   __lowercase			):
  '''simple docstring'''







  SCREAMING_SNAKE_CASE							:				Union[str, Any]					      =				VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE							:				Any					      =				PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE							:				Dict					      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE							:				List[str]					      =				['''input_ids''', '''attention_mask''']
  SCREAMING_SNAKE_CASE							:				Tuple					      =				OpenAIGPTTokenizer







  def __init__( self	: Dict		,						UpperCamelCase__	: List[Any]=None		,						UpperCamelCase__	: Union[str, Any]=None		,						UpperCamelCase__	: Optional[int]=None		,						UpperCamelCase__	: Tuple="<unk>"		,						**UpperCamelCase__	: Union[str, Any] ):
        super().__init__(UpperCamelCase__		,						UpperCamelCase__		,						tokenizer_file=UpperCamelCase__		,						unk_token=UpperCamelCase__		,						**UpperCamelCase__ )







  @property
  def 	UpperCamelCase       ( self	: Optional[Any] ):
        return True







  def 	UpperCamelCase       ( self	: Dict		,						UpperCamelCase__	: str		,						UpperCamelCase__	: Optional[str] = None ):
        A    =				self._tokenizer.model.save(UpperCamelCase__		,						name=UpperCamelCase__ )
        return tuple(UpperCamelCase__ )



