
from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A   =     logging.get_logger(__name__)

_A   =     {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/config.json"}





class 	_lowerCAmelCase					(	__a       ):
				_lowercase					='''openai-gpt'''
				_lowercase					={
				    '''max_position_embeddings''': '''n_positions''',
				    '''hidden_size''': '''n_embd''',
				    '''num_attention_heads''': '''n_head''',
				    '''num_hidden_layers''': '''n_layer''',
				}




				def __init__(					self				,				_UpperCamelCase=40_478				,				_UpperCamelCase=512				,				_UpperCamelCase=768				,				_UpperCamelCase=12				,				_UpperCamelCase=12				,				_UpperCamelCase="gelu"				,				_UpperCamelCase=0.1				,				_UpperCamelCase=0.1				,				_UpperCamelCase=0.1				,				_UpperCamelCase=1e-5				,				_UpperCamelCase=0.02				,				_UpperCamelCase="cls_index"				,				_UpperCamelCase=True				,				_UpperCamelCase=None				,				_UpperCamelCase=True				,				_UpperCamelCase=0.1				,				**_UpperCamelCase				,				)				->      str:
							lowerCAmelCase_						=	vocab_size
							lowerCAmelCase_						=	n_positions
							lowerCAmelCase_						=	n_embd
							lowerCAmelCase_						=	n_layer
							lowerCAmelCase_						=	n_head
							lowerCAmelCase_						=	afn
							lowerCAmelCase_						=	resid_pdrop
							lowerCAmelCase_						=	embd_pdrop
							lowerCAmelCase_						=	attn_pdrop
							lowerCAmelCase_						=	layer_norm_epsilon
							lowerCAmelCase_						=	initializer_range
							lowerCAmelCase_						=	summary_type
							lowerCAmelCase_						=	summary_use_proj
							lowerCAmelCase_						=	summary_activation
							lowerCAmelCase_						=	summary_first_dropout
							lowerCAmelCase_						=	summary_proj_to_labels
							super().__init__(**_UpperCamelCase     )





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 	_lowerCAmelCase					(	__a					,      unittest.TestCase       ):
				_lowercase					=CLIPTokenizer
				_lowercase					=CLIPTokenizerFast
				_lowercase					=True
				_lowercase					={}
				_lowercase					=False




				def      __a						(					self     )				->      Dict:
							super().setUp()

							# fmt: off
							lowerCAmelCase_						=	["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
							# fmt: on
							lowerCAmelCase_						=	dict(zip(_UpperCamelCase				,				range(len(_UpperCamelCase     )     )     )     )
							lowerCAmelCase_						=	["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
							lowerCAmelCase_						=	{"unk_token": "<unk>"}

							lowerCAmelCase_						=	os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["vocab_file"]     )
							lowerCAmelCase_						=	os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["merges_file"]     )
							with open(self.vocab_file				,				"w"				,				encoding="utf-8"     ) as fp:
										fp.write(json.dumps(_UpperCamelCase     ) + "\n"     )
							with open(self.merges_file				,				"w"				,				encoding="utf-8"     ) as fp:
										fp.write("\n".join(_UpperCamelCase     )     )




				def      __a						(					self				,				**_UpperCamelCase     )				->      Any:
							kwargs.update(self.special_tokens_map     )
							return CLIPTokenizer.from_pretrained(self.tmpdirname				,				**_UpperCamelCase     )




				def      __a						(					self				,				**_UpperCamelCase     )				->      int:
							kwargs.update(self.special_tokens_map     )
							return CLIPTokenizerFast.from_pretrained(self.tmpdirname				,				**_UpperCamelCase     )




				def      __a						(					self				,				_UpperCamelCase     )				->      List[str]:
							lowerCAmelCase_						=	"lower newer"
							lowerCAmelCase_						=	"lower newer"
							return input_text, output_text




				def      __a						(					self     )				->      List[Any]:
							lowerCAmelCase_						=	CLIPTokenizer(self.vocab_file				,				self.merges_file				,				**self.special_tokens_map     )
							lowerCAmelCase_						=	"lower newer"
							lowerCAmelCase_						=	["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
							lowerCAmelCase_						=	tokenizer.tokenize(_UpperCamelCase     )
							self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )

							lowerCAmelCase_						=	tokens + [tokenizer.unk_token]
							lowerCAmelCase_						=	[10, 2, 16, 9, 3, 2, 16, 20]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase     )				,				_UpperCamelCase     )




				@require_ftfy
				def      __a						(					self     )				->      Union[str, Any]:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
													lowerCAmelCase_						=	self.tokenizer_class.from_pretrained(_UpperCamelCase				,				**_UpperCamelCase     )
													lowerCAmelCase_						=	self.rust_tokenizer_class.from_pretrained(_UpperCamelCase				,				**_UpperCamelCase     )

													lowerCAmelCase_						=	"A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
													lowerCAmelCase_						=	tokenizer_s.tokenize(_UpperCamelCase     )
													lowerCAmelCase_						=	tokenizer_r.tokenize(_UpperCamelCase     )

													self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )

													# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
													# with Tilde) encoded in 2 different ways
													lowerCAmelCase_						=	"xa\u0303y" + " " + "x\xe3y"
													lowerCAmelCase_						=	tokenizer_s.tokenize(_UpperCamelCase     )
													lowerCAmelCase_						=	tokenizer_r.tokenize(_UpperCamelCase     )

													self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )

													# Test that the tokenization is identical on unicode of space type
													lowerCAmelCase_						=	[
													    "\u0009",  # (horizontal tab, '\t')
													    "\u000B",  # (vertical tab)
													    "\u000C",  # (form feed)
													    "\u0020",  # (space, ' ')
													    "\u200E",  # (left-to-right mark):w
													    "\u200F",  # (right-to-left mark)
													]
													for unicode_seq in spaces_unicodes:
																lowerCAmelCase_						=	tokenizer_s.tokenize(_UpperCamelCase     )
																lowerCAmelCase_						=	tokenizer_r.tokenize(_UpperCamelCase     )

																self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )

													# Test that the tokenization is identical on unicode of line break type
													lowerCAmelCase_						=	[
													    "\u000A",  # (line feed, '\n')
													    "\r\n",  # (carriage return and line feed, '\r\n')
													    "\u000D",  # (carriage return, '\r')
													    "\r",  # (carriage return, '\r')
													    "\u000D",  # (carriage return, '\r')
													    "\u2028",  # (line separator)
													    "\u2029",  # (paragraph separator)
													    # "\u0085", # (next line)
													]

													# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
													# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
													# space (and thus into an empty list).

													for unicode_seq in line_break_unicodes:
																lowerCAmelCase_						=	tokenizer_s.tokenize(_UpperCamelCase     )
																lowerCAmelCase_						=	tokenizer_r.tokenize(_UpperCamelCase     )

																self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )




				def      __a						(					self     )				->      str:
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
													lowerCAmelCase_						=	"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
													lowerCAmelCase_						=	f"""{text_of_1_token} {text_of_1_token}"""

													lowerCAmelCase_						=	self.rust_tokenizer_class.from_pretrained(
													    _UpperCamelCase				,				use_fast=_UpperCamelCase				,				)
													lowerCAmelCase_						=	tokenizer_r(_UpperCamelCase				,				return_offsets_mapping=_UpperCamelCase				,				add_special_tokens=_UpperCamelCase     )
													self.assertEqual(encoding.offset_mapping[0]				,				(0, len(_UpperCamelCase     ))     )
													self.assertEqual(
													    encoding.offset_mapping[1]				,				(len(_UpperCamelCase     ) + 1, len(_UpperCamelCase     ) + 1 + len(_UpperCamelCase     ))				,				)

													lowerCAmelCase_						=	f""" {text}"""

													lowerCAmelCase_						=	self.rust_tokenizer_class.from_pretrained(
													    _UpperCamelCase				,				use_fast=_UpperCamelCase				,				)
													lowerCAmelCase_						=	tokenizer_r(_UpperCamelCase				,				return_offsets_mapping=_UpperCamelCase				,				add_special_tokens=_UpperCamelCase     )
													self.assertEqual(encoding.offset_mapping[0]				,				(1, 1 + len(_UpperCamelCase     ))     )
													self.assertEqual(
													    encoding.offset_mapping[1]				,				(1 + len(_UpperCamelCase     ) + 1, 1 + len(_UpperCamelCase     ) + 1 + len(_UpperCamelCase     ))				,				)




				def      __a						(					self     )				->      Optional[Any]:
							# Test related to the breaking change introduced in transformers v4.17.0
							# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
							with self.assertRaises(_UpperCamelCase     ) as context:
										self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"     )

							self.assertTrue(
							    context.exception.args[0].startswith(
							        "The `backend_tokenizer` provided does not match the expected format."     )     )




				@require_ftfy
				def      __a						(					self     )				->      str:
							super().test_tokenization_python_rust_equals()




				def      __a						(					self     )				->      Any:
							# CLIP always lower cases letters
							pass



