
'''simple docstring'''







import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_snake_case					: Optional[Any]     			=					logging.get_logger(__name__)

_snake_case					: Optional[Any]     			=					{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

_snake_case					: Any     			=					{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

_snake_case					: Optional[Any]     			=					{
    """openai-gpt""": 512,
}



def       _a		(						_SCREAMING_SNAKE_CASE		: Union[str, Any]	):
						_SCREAMING_SNAKE_CASE					=			text.strip()
						if not text:
												return []
						_SCREAMING_SNAKE_CASE					=			text.split()
						return tokens


class       lowerCAmelCase			(					__UpperCAmelCase      ):




		def __init__(					self       ,    UpperCamelCase=True       ,    UpperCamelCase=None       ,    UpperCamelCase=True       ,    UpperCamelCase=None       ,    UpperCamelCase=True       ,    ):
								if never_split is None:
														_SCREAMING_SNAKE_CASE					=			[]
								_SCREAMING_SNAKE_CASE					=			do_lower_case
								_SCREAMING_SNAKE_CASE					=			set(UpperCamelCase   )
								_SCREAMING_SNAKE_CASE					=			tokenize_chinese_chars
								_SCREAMING_SNAKE_CASE					=			strip_accents
								_SCREAMING_SNAKE_CASE					=			do_split_on_punc




		def      lowercase							(					self       ,    UpperCamelCase       ,    UpperCamelCase=None   ):
								_SCREAMING_SNAKE_CASE					=			self.never_split.union(set(UpperCamelCase   )   ) if never_split else self.never_split
								_SCREAMING_SNAKE_CASE					=			self._clean_text(UpperCamelCase   )

								# This was added on November 1st, 2018 for the multilingual and Chinese
								# models. This is also applied to the English models now, but it doesn't
								# matter since the English models were not trained on any Chinese data
								# and generally don't have any Chinese data in them (there are Chinese
								# characters in the vocabulary because Wikipedia does have some Chinese
								# words in the English Wikipedia.).
								if self.tokenize_chinese_chars:
														_SCREAMING_SNAKE_CASE					=			self._tokenize_chinese_chars(UpperCamelCase   )
								# prevents treating the same character with different unicode codepoints as different characters
								_SCREAMING_SNAKE_CASE					=			unicodedata.normalize("NFC"       ,    UpperCamelCase   )
								_SCREAMING_SNAKE_CASE					=			whitespace_tokenize(UpperCamelCase   )
								_SCREAMING_SNAKE_CASE					=			[]
								for token in orig_tokens:
														if token not in never_split:
																				if self.do_lower_case:
																										_SCREAMING_SNAKE_CASE					=			token.lower()
																										if self.strip_accents is not False:
																																_SCREAMING_SNAKE_CASE					=			self._run_strip_accents(UpperCamelCase   )
																				elif self.strip_accents:
																										_SCREAMING_SNAKE_CASE					=			self._run_strip_accents(UpperCamelCase   )
														split_tokens.extend(self._run_split_on_punc(UpperCamelCase       ,    UpperCamelCase   )   )

								_SCREAMING_SNAKE_CASE					=			whitespace_tokenize(" ".join(UpperCamelCase   )   )
								return output_tokens




		def      lowercase							(					self       ,    UpperCamelCase   ):
								_SCREAMING_SNAKE_CASE					=			unicodedata.normalize("NFD"       ,    UpperCamelCase   )
								_SCREAMING_SNAKE_CASE					=			[]
								for char in text:
														_SCREAMING_SNAKE_CASE					=			unicodedata.category(UpperCamelCase   )
														if cat == "Mn":
																				continue
														output.append(UpperCamelCase   )
								return "".join(UpperCamelCase   )




		def      lowercase							(					self       ,    UpperCamelCase       ,    UpperCamelCase=None   ):
								if not self.do_split_on_punc or (never_split is not None and text in never_split):
														return [text]
								_SCREAMING_SNAKE_CASE					=			list(UpperCamelCase   )
								_SCREAMING_SNAKE_CASE					=			0
								_SCREAMING_SNAKE_CASE					=			True
								_SCREAMING_SNAKE_CASE					=			[]
								while i < len(UpperCamelCase   ):
														_SCREAMING_SNAKE_CASE					=			chars[i]
														if _is_punctuation(UpperCamelCase   ):
																				output.append([char]   )
																				_SCREAMING_SNAKE_CASE					=			True
														else:
																				if start_new_word:
																										output.append([]   )
																				_SCREAMING_SNAKE_CASE					=			False
																				output[-1].append(UpperCamelCase   )
														i += 1

								return ["".join(UpperCamelCase   ) for x in output]




		def      lowercase							(					self       ,    UpperCamelCase   ):
								_SCREAMING_SNAKE_CASE					=			[]
								for char in text:
														_SCREAMING_SNAKE_CASE					=			ord(UpperCamelCase   )
														if self._is_chinese_char(UpperCamelCase   ):
																				output.append(" "   )
																				output.append(UpperCamelCase   )
																				output.append(" "   )
														else:
																				output.append(UpperCamelCase   )
								return "".join(UpperCamelCase   )




		def      lowercase							(					self       ,    UpperCamelCase   ):
								# This defines a "chinese character" as anything in the CJK Unicode block:
								#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
								#
								# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
								# despite its name. The modern Korean Hangul alphabet is a different block,
								# as is Japanese Hiragana and Katakana. Those alphabets are used to write
								# space-separated words, so they are not treated specially and handled
								# like the all of the other languages.
								if (
								    (cp >= 0x4E_00 and cp <= 0x9F_FF)
								    or (cp >= 0x34_00 and cp <= 0x4D_BF)  #
								    or (cp >= 0x2_00_00 and cp <= 0x2_A6_DF)  #
								    or (cp >= 0x2_A7_00 and cp <= 0x2_B7_3F)  #
								    or (cp >= 0x2_B7_40 and cp <= 0x2_B8_1F)  #
								    or (cp >= 0x2_B8_20 and cp <= 0x2_CE_AF)  #
								    or (cp >= 0xF9_00 and cp <= 0xFA_FF)
								    or (cp >= 0x2_F8_00 and cp <= 0x2_FA_1F)  #
								):  #
														return True

								return False




		def      lowercase							(					self       ,    UpperCamelCase   ):
								_SCREAMING_SNAKE_CASE					=			[]
								for char in text:
														_SCREAMING_SNAKE_CASE					=			ord(UpperCamelCase   )
														if cp == 0 or cp == 0xFF_FD or _is_control(UpperCamelCase   ):
																				continue
														if _is_whitespace(UpperCamelCase   ):
																				output.append(" "   )
														else:
																				output.append(UpperCamelCase   )
								return "".join(UpperCamelCase   )



def       _a		(						_SCREAMING_SNAKE_CASE		: List[Any]	):
						_SCREAMING_SNAKE_CASE					=			set()
						_SCREAMING_SNAKE_CASE					=			word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)	)
												_SCREAMING_SNAKE_CASE					=			char
						return pairs



def       _a		(						_SCREAMING_SNAKE_CASE		: Dict	):
						_SCREAMING_SNAKE_CASE					=			text.replace("—"						,      "-"	)
						_SCREAMING_SNAKE_CASE					=			text.replace("–"						,      "-"	)
						_SCREAMING_SNAKE_CASE					=			text.replace("―"						,      "-"	)
						_SCREAMING_SNAKE_CASE					=			text.replace("…"						,      "..."	)
						_SCREAMING_SNAKE_CASE					=			text.replace("´"						,      "'"	)
						_SCREAMING_SNAKE_CASE					=			re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"						,      r" \1 "						,      _SCREAMING_SNAKE_CASE	)
						_SCREAMING_SNAKE_CASE					=			re.sub(r"\s*\n\s*"						,      " \n "						,      _SCREAMING_SNAKE_CASE	)
						_SCREAMING_SNAKE_CASE					=			re.sub(r"[^\S\n]+"						,      " "						,      _SCREAMING_SNAKE_CASE	)
						return text.strip()




class       lowerCAmelCase			(					__UpperCAmelCase      ):
		a				:       Optional[Any]							=       VOCAB_FILES_NAMES
		a				:       str							=       PRETRAINED_VOCAB_FILES_MAP
		a				:       Union[str, Any]							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a				:       List[str]							=       ["""input_ids""", """attention_mask"""]




		def __init__(					self       ,    UpperCamelCase       ,    UpperCamelCase       ,    UpperCamelCase="<unk>"       ,    **UpperCamelCase   ):
								super().__init__(unk_token=UpperCamelCase       ,    **UpperCamelCase   )

								try:
														import ftfy
														from spacy.lang.en import English

														_SCREAMING_SNAKE_CASE					=			English()
														_SCREAMING_SNAKE_CASE					=			_nlp.tokenizer
														_SCREAMING_SNAKE_CASE					=			ftfy.fix_text
								except ImportError:
														logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."   )
														_SCREAMING_SNAKE_CASE					=			BasicTokenizer(do_lower_case=UpperCamelCase   )
														_SCREAMING_SNAKE_CASE					=			None

								with open(UpperCamelCase       ,    encoding="utf-8"   ) as vocab_handle:
														_SCREAMING_SNAKE_CASE					=			json.load(UpperCamelCase   )
								_SCREAMING_SNAKE_CASE					=			{v: k for k, v in self.encoder.items()}
								with open(UpperCamelCase       ,    encoding="utf-8"   ) as merges_handle:
														_SCREAMING_SNAKE_CASE					=			merges_handle.read().split("\n"   )[1:-1]
								_SCREAMING_SNAKE_CASE					=			[tuple(merge.split()   ) for merge in merges]
								_SCREAMING_SNAKE_CASE					=			dict(zip(UpperCamelCase       ,    range(len(UpperCamelCase   )   )   )   )
								_SCREAMING_SNAKE_CASE					=			{}




		@property
		def      lowercase							(					self   ):
								return True




		@property
		def      lowercase							(					self   ):
								return len(self.encoder   )




		def      lowercase							(					self   ):
								return dict(self.encoder       ,    **self.added_tokens_encoder   )




		def      lowercase							(					self       ,    UpperCamelCase   ):
								_SCREAMING_SNAKE_CASE					=			tuple(token[:-1]   ) + (token[-1] + "</w>",)
								if token in self.cache:
														return self.cache[token]
								_SCREAMING_SNAKE_CASE					=			get_pairs(UpperCamelCase   )

								if not pairs:
														return token + "</w>"

								while True:
														_SCREAMING_SNAKE_CASE					=			min(UpperCamelCase       ,    key=lambda UpperCamelCase   : self.bpe_ranks.get(UpperCamelCase       ,    float("inf"   )   )   )
														if bigram not in self.bpe_ranks:
																				break
														_SCREAMING_SNAKE_CASE,				_SCREAMING_SNAKE_CASE					=			bigram
														_SCREAMING_SNAKE_CASE					=			[]
														_SCREAMING_SNAKE_CASE					=			0
														while i < len(UpperCamelCase   ):
																				try:
																										_SCREAMING_SNAKE_CASE					=			word.index(UpperCamelCase       ,    UpperCamelCase   )
																				except ValueError:
																										new_word.extend(word[i:]   )
																										break
																				else:
																										new_word.extend(word[i:j]   )
																										_SCREAMING_SNAKE_CASE					=			j

																				if word[i] == first and i < len(UpperCamelCase   ) - 1 and word[i + 1] == second:
																										new_word.append(first + second   )
																										i += 2
																				else:
																										new_word.append(word[i]   )
																										i += 1
														_SCREAMING_SNAKE_CASE					=			tuple(UpperCamelCase   )
														_SCREAMING_SNAKE_CASE					=			new_word
														if len(UpperCamelCase   ) == 1:
																				break
														else:
																				_SCREAMING_SNAKE_CASE					=			get_pairs(UpperCamelCase   )
								_SCREAMING_SNAKE_CASE					=			" ".join(UpperCamelCase   )
								if word == "\n  </w>":
														_SCREAMING_SNAKE_CASE					=			"\n</w>"
								_SCREAMING_SNAKE_CASE					=			word
								return word




		def      lowercase							(					self       ,    UpperCamelCase   ):
								_SCREAMING_SNAKE_CASE					=			[]
								if self.fix_text is None:
														# Using BERT's BasicTokenizer
														_SCREAMING_SNAKE_CASE					=			self.nlp.tokenize(UpperCamelCase   )
														for token in text:
																				split_tokens.extend(list(self.bpe(UpperCamelCase   ).split(" "   )   )   )
								else:
														# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
														_SCREAMING_SNAKE_CASE					=			self.nlp(text_standardize(self.fix_text(UpperCamelCase   )   )   )
														for token in text:
																				split_tokens.extend(list(self.bpe(token.text.lower()   ).split(" "   )   )   )
								return split_tokens




		def      lowercase							(					self       ,    UpperCamelCase   ):
								return self.encoder.get(UpperCamelCase       ,    self.encoder.get(self.unk_token   )   )




		def      lowercase							(					self       ,    UpperCamelCase   ):
								return self.decoder.get(UpperCamelCase       ,    self.unk_token   )




		def      lowercase							(					self       ,    UpperCamelCase   ):
								_SCREAMING_SNAKE_CASE					=			"".join(UpperCamelCase   ).replace("</w>"       ,    " "   ).strip()
								return out_string




		def      lowercase							(					self       ,    UpperCamelCase       ,    UpperCamelCase = None   ):
								if not os.path.isdir(UpperCamelCase   ):
														logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
														return
								_SCREAMING_SNAKE_CASE					=			os.path.join(
								    UpperCamelCase       ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
								_SCREAMING_SNAKE_CASE					=			os.path.join(
								    UpperCamelCase       ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )

								with open(UpperCamelCase       ,    "w"       ,    encoding="utf-8"   ) as f:
														f.write(json.dumps(self.encoder       ,    indent=2       ,    sort_keys=UpperCamelCase       ,    ensure_ascii=UpperCamelCase   ) + "\n"   )

								_SCREAMING_SNAKE_CASE					=			0
								with open(UpperCamelCase       ,    "w"       ,    encoding="utf-8"   ) as writer:
														writer.write("#version: 0.2\n"   )
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,    key=lambda UpperCamelCase   : kv[1]   ):
																				if index != token_index:
																										logger.warning(
																										    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																										    " Please check that the tokenizer is not corrupted!"   )
																										_SCREAMING_SNAKE_CASE					=			token_index
																				writer.write(" ".join(UpperCamelCase   ) + "\n"   )
																				index += 1

								return vocab_file, merge_file

'''simple docstring'''







import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
						import torch

						from transformers import GPTaLMHeadModel


@require_torch
@require_sentencepiece
@require_tokenizers
class       lowerCAmelCase			(					unittest.TestCase      ):




		@slow
		@unittest.skip("Model is not available."   )
		def      lowercase							(					self   ):
								_SCREAMING_SNAKE_CASE					=			"nvidia/megatron-gpt2-345m/"
								if "MYDIR" in os.environ:
														_SCREAMING_SNAKE_CASE					=			os.path.join(os.environ["MYDIR"]       ,    UpperCamelCase   )
								_SCREAMING_SNAKE_CASE					=			GPTaLMHeadModel.from_pretrained(UpperCamelCase   )
								model.to(UpperCamelCase   )
								model.half()

								_SCREAMING_SNAKE_CASE					=			torch.tensor(
								    [[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]       ,    device=UpperCamelCase       ,    dtype=torch.long       ,    )

								with torch.no_grad():
														_SCREAMING_SNAKE_CASE					=			model(UpperCamelCase   ).logits

								_SCREAMING_SNAKE_CASE					=			torch.Size((1, 9, 50_257)   )
								self.assertEqual(output.shape       ,    UpperCamelCase   )

								_SCREAMING_SNAKE_CASE					=			torch.tensor(
								    [
								        4.94_14,
								        -0.29_20,
								        -1.21_48,
								        -4.02_73,
								        -0.51_61,
								        -5.21_09,
								        -1.24_12,
								        -1.83_01,
								        -1.77_34,
								        -4.71_48,
								        -0.23_17,
								        -1.08_11,
								        -2.17_77,
								        0.41_41,
								        -3.79_69,
								        -4.05_86,
								        -2.53_32,
								        -3.38_09,
								        4.38_67,
								    ]       ,    device=UpperCamelCase       ,    dtype=torch.half       ,    )

								for i in range(19   ):
														_SCREAMING_SNAKE_CASE,				_SCREAMING_SNAKE_CASE					=			8 * i // 17, 2_792 * i  # along the diagonal
														_SCREAMING_SNAKE_CASE,				_SCREAMING_SNAKE_CASE					=			output[0, r, c], expected_diag[i]
														_SCREAMING_SNAKE_CASE					=			F'row={r} col={c} computed={computed} expected={expected}'
														self.assertAlmostEqual(UpperCamelCase       ,    UpperCamelCase       ,    delta=1e-4       ,    msg=UpperCamelCase   )