






"""simple docstring"""



from __future__ import annotations

import unittest

from transformers import OpenAIGPTConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers.models.openai.modeling_tf_openai import (
        TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFOpenAIGPTDoubleHeadsModel,
        TFOpenAIGPTForSequenceClassification,
        TFOpenAIGPTLMHeadModel,
        TFOpenAIGPTModel,
    )

class 			a	:







    def __init__(     self				,						_lowerCamelCase				,						):
      lowercase				       =			parent
      lowercase				       =			1_3
      lowercase				       =			7
      lowercase				       =			True
      lowercase				       =			True
      lowercase				       =			True
      lowercase				       =			True
      lowercase				       =			True
      lowercase				       =			9_9
      lowercase				       =			3_2
      lowercase				       =			2
      lowercase				       =			4
      lowercase				       =			3_7
      lowercase				       =			'gelu'
      lowercase				       =			0.1
      lowercase				       =			0.1
      lowercase				       =			5_1_2
      lowercase				       =			1_6
      lowercase				       =			2
      lowercase				       =			0.0_2
      lowercase				       =			3
      lowercase				       =			4
      lowercase				       =			None
      lowercase				       =			self.vocab_size - 1







    def      UpperCamelCase_ (     self			):
      lowercase				       =			ids_tensor([self.batch_size, self.seq_length]				,						self.vocab_size			)

      lowercase				       =			None
      if self.use_input_mask:
        lowercase				       =			random_attention_mask([self.batch_size, self.seq_length]			)

      lowercase				       =			None
      if self.use_token_type_ids:
        lowercase				       =			ids_tensor([self.batch_size, self.seq_length]				,						self.type_vocab_size			)

      lowercase				       =			None
      if self.use_mc_token_ids:
        lowercase				       =			ids_tensor([self.batch_size, self.num_choices]				,						self.seq_length			)

      lowercase				       =			None
      lowercase				       =			None
      lowercase				       =			None
      if self.use_labels:
        lowercase				       =			ids_tensor([self.batch_size]				,						self.type_sequence_label_size			)
        lowercase				       =			ids_tensor([self.batch_size, self.seq_length]				,						self.num_labels			)
        lowercase				       =			ids_tensor([self.batch_size]				,						self.num_choices			)

      lowercase				       =			OpenAIGPTConfig(
          vocab_size=self.vocab_size				,						n_embd=self.hidden_size				,						n_layer=self.num_hidden_layers				,						n_head=self.num_attention_heads				,						n_positions=self.max_position_embeddings				,						pad_token_id=self.pad_token_id				,						)

      lowercase				       =			ids_tensor([self.num_hidden_layers, self.num_attention_heads]				,						2			)

      return (
          config,
          input_ids,
          input_mask,
          head_mask,
          token_type_ids,
          mc_token_ids,
          sequence_labels,
          token_labels,
          choice_labels,
      )







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						*_lowerCamelCase			):
      lowercase				       =			TFOpenAIGPTModel(config=_lowerCamelCase			)
      lowercase				       =			{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
      lowercase				       =			model(_lowerCamelCase			)

      lowercase				       =			[input_ids, input_mask]
      lowercase				       =			model(_lowerCamelCase			)

      lowercase				       =			model(_lowerCamelCase			)

      self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.hidden_size)			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						*_lowerCamelCase			):
      lowercase				       =			TFOpenAIGPTLMHeadModel(config=_lowerCamelCase			)
      lowercase				       =			{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
      lowercase				       =			model(_lowerCamelCase			)
      self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length, self.vocab_size)			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						*_lowerCamelCase			):
      lowercase				       =			TFOpenAIGPTDoubleHeadsModel(config=_lowerCamelCase			)

      lowercase				       =			tf.tile(tf.expand_dims(_lowerCamelCase				,						1			)				,						(1, self.num_choices, 1)			)
      lowercase				       =			tf.tile(tf.expand_dims(_lowerCamelCase				,						1			)				,						(1, self.num_choices, 1)			)
      lowercase				       =			tf.tile(tf.expand_dims(_lowerCamelCase				,						1			)				,						(1, self.num_choices, 1)			)

      lowercase				       =			{
          'input_ids': multiple_choice_inputs_ids,
          'mc_token_ids': mc_token_ids,
          'attention_mask': multiple_choice_input_mask,
          'token_type_ids': multiple_choice_token_type_ids,
      }
      lowercase				       =			model(_lowerCamelCase			)
      self.parent.assertEqual(
          result.logits.shape				,						(self.batch_size, self.num_choices, self.seq_length, self.vocab_size)			)
      self.parent.assertEqual(result.mc_logits.shape				,						(self.batch_size, self.num_choices)			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						*_lowerCamelCase			):
      lowercase				       =			self.num_labels
      lowercase				       =			ids_tensor([self.batch_size]				,						self.type_sequence_label_size			)
      lowercase				       =			{
          'input_ids': input_ids,
          'attention_mask': input_mask,
          'token_type_ids': token_type_ids,
          'labels': sequence_labels,
      }
      lowercase				       =			TFOpenAIGPTForSequenceClassification(_lowerCamelCase			)
      lowercase				       =			model(_lowerCamelCase			)
      self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.num_labels)			)







    def      UpperCamelCase_ (     self			):
      lowercase				       =			self.prepare_config_and_inputs()

      (
          (
          lowercase
      )       ,	(
          lowercase
      )       ,	(
          lowercase
      )       ,	(
          lowercase
      )       ,	(
          lowercase
      )       ,	(
          lowercase
      )       ,	(
          lowercase
      )       ,	(
          lowercase
      )       ,	(
          lowercase
      )       ,	
      )				       =			config_and_inputs

      lowercase				       =			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
      return config, inputs_dict


@require_tf
class 			a	(  a_,							a_,							unittest.TestCase      ):
    UpperCAmelCase_					:	Any      	=(
        (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TFOpenAIGPTForSequenceClassification)
        if is_tf_available()
        else ()
    )
    UpperCAmelCase_					:	int      	=(
        (TFOpenAIGPTLMHeadModel,) if is_tf_available() else ()
    )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
    UpperCAmelCase_					:	str      	=(
        {
            "feature-extraction": TFOpenAIGPTModel,
            "text-classification": TFOpenAIGPTForSequenceClassification,
            "text-generation": TFOpenAIGPTLMHeadModel,
            "zero-shot": TFOpenAIGPTForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )
    UpperCAmelCase_					:	List[str]      	=False
    UpperCAmelCase_					:	Optional[int]      	=False







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase			):
      if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
        # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
        # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
        # tiny config could not be created.
        return True

      return False







    def      UpperCamelCase_ (     self			):
      lowercase				       =			TFOpenAIGPTModelTester(self			)
      lowercase				       =			ConfigTester(self				,						config_class=_lowerCamelCase				,						n_embd=3_7			)







    def      UpperCamelCase_ (     self			):
      self.config_tester.run_common_tests()







    def      UpperCamelCase_ (     self			):
      lowercase				       =			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_openai_gpt_model(*_lowerCamelCase			)







    def      UpperCamelCase_ (     self			):
      lowercase				       =			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_openai_gpt_lm_head(*_lowerCamelCase			)







    def      UpperCamelCase_ (     self			):
      lowercase				       =			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_openai_gpt_double_head(*_lowerCamelCase			)







    def      UpperCamelCase_ (     self			):
      lowercase				       =			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*_lowerCamelCase			)







    @slow
    def      UpperCamelCase_ (     self			):
      for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        lowercase				       =			TFOpenAIGPTModel.from_pretrained(_lowerCamelCase			)
        self.assertIsNotNone(_lowerCamelCase			)


@require_tf
class 			a	(  unittest.TestCase      ):







    @slow
    def      UpperCamelCase_ (     self			):
      lowercase				       =			TFOpenAIGPTLMHeadModel.from_pretrained('openai-gpt'			)
      lowercase				       =			tf.convert_to_tensor([[4_8_1, 4_7_3_5, 5_4_4]]				,						dtype=tf.intaa			)  # the president is
      lowercase				       =			[
          4_8_1,
          4_7_3_5,
          5_4_4,
          2_4_6,
          9_6_3,
          8_7_0,
          7_6_2,
          2_3_9,
          2_4_4,
          4_0_4_7_7,
          2_4_4,
          2_4_9,
          7_1_9,
          8_8_1,
          4_8_7,
          5_4_4,
          2_4_0,
          2_4_4,
          6_0_3,
          4_8_1,
      ]  # the president is a very good man. " \n " i\'m sure he is, " said the

      lowercase				       =			model.generate(_lowerCamelCase				,						do_sample=_lowerCamelCase			)
      self.assertListEqual(output_ids[0].numpy().tolist()				,						_lowerCamelCase			)








'''simple docstring'''







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 	lowerCamelCase_				(			UpperCAmelCase_,			unittest.TestCase				):




					'''simple docstring'''






					lowercase__       =					PreTrainedTokenizerFast
					lowercase__       =					False
					lowercase__       =					True
					lowercase__       =					"""tokenizer_file"""






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			False  # because we don't have pretrained_vocab_files_map
							super().setUp()
							_snake_case      			=			True

							_snake_case      			=			["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
							_snake_case      			=			"SaulLu/dummy-tokenizer-bytelevel-bpe"

							# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
							_snake_case      			=			[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

							_snake_case      			=			PreTrainedTokenizerFast.from_pretrained(model_paths[0]					)
							tokenizer.save_pretrained(self.tmpdirname					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
							# model
							pass






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
							# model
							pass






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
							# model
							pass






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
							pass






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.tmpdirname
							# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
											try:
													_snake_case      			=			tempfile.mkdtemp()
													_snake_case      			=			self.rust_tokenizer_class.from_pretrained(__UpperCamelCase,    **__UpperCamelCase					)

													tokenizer.save_pretrained(self.tmpdirname					)
													super().test_training_new_tokenizer()
											finally:
													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
													# is restored
													shutil.rmtree(self.tmpdirname					)
													_snake_case      			=			tmpdirname_orig






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.tmpdirname
							# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
											try:
													_snake_case      			=			tempfile.mkdtemp()
													_snake_case      			=			self.rust_tokenizer_class.from_pretrained(__UpperCamelCase,    **__UpperCamelCase					)

													tokenizer.save_pretrained(self.tmpdirname					)
													super().test_training_new_tokenizer_with_special_tokens_change()
											finally:
													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
													# is restored
													shutil.rmtree(self.tmpdirname					)
													_snake_case      			=			tmpdirname_orig






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name					)

							_snake_case      			=			("a" for _ in range(1_000					))
							_snake_case      			=			tokenizer.train_new_from_iterator(text_iterator=__UpperCamelCase,    length=1_000,    vocab_size=50					)

							_snake_case      			=			new_tokenizer.encode("a🤗"					)
							self.assertEqual(__UpperCamelCase,    [64, 172, 253, 97, 245]					)





@require_tokenizers
class 	lowerCamelCase_				(			unittest.TestCase				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			AutoTokenizer.from_pretrained("bert-base-cased"					)
							_snake_case      			=			json.loads(tokenizer._tokenizer.to_str()					)
							_snake_case      			=			len(__UpperCamelCase					)

							with tempfile.TemporaryDirectory() as tmp_dir:
									# Hack to save this in the tokenizer_config.json
									_snake_case      			=			["tokenizer.4.0.0.json"]
									tokenizer.save_pretrained(__UpperCamelCase					)
									json.dump(__UpperCamelCase,    open(os.path.join(__UpperCamelCase,    "tokenizer.4.0.0.json"					),    "w"					)					)

									# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
									_snake_case      			=			AutoTokenizer.from_pretrained(__UpperCamelCase					)
									self.assertEqual(len(__UpperCamelCase					),    len(__UpperCamelCase					) + 1					)
									_snake_case      			=			json.loads(new_tokenizer._tokenizer.to_str()					)
									self.assertIn("huggingface",    json_tokenizer["model"]["vocab"]					)

									# Will need to be adjusted if we reach v42 and this test is still here.
									# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
									shutil.move(os.path.join(__UpperCamelCase,    "tokenizer.4.0.0.json"					),    os.path.join(__UpperCamelCase,    "tokenizer.42.0.0.json"					)					)
									_snake_case      			=			["tokenizer.42.0.0.json"]
									tokenizer.save_pretrained(__UpperCamelCase					)
									_snake_case      			=			AutoTokenizer.from_pretrained(__UpperCamelCase					)
									self.assertEqual(len(__UpperCamelCase					),    len(__UpperCamelCase					)					)
									_snake_case      			=			json.loads(new_tokenizer._tokenizer.to_str()					)
									self.assertNotIn("huggingface",    json_tokenizer["model"]["vocab"]					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
							_snake_case      			=			"hf-internal-testing/test-two-tokenizers"

							# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
							_snake_case      			=			AutoTokenizer.from_pretrained(__UpperCamelCase					)
							self.assertEqual(len(__UpperCamelCase					),    28_997					)
							_snake_case      			=			json.loads(tokenizer._tokenizer.to_str()					)
							self.assertIn("huggingface",    json_tokenizer["model"]["vocab"]					)

							# Testing an older version by monkey-patching the version in the module it's used.
							import transformers as old_transformers

							_snake_case      			=			"3.0.0"
							_snake_case      			=			old_transformers.models.auto.AutoTokenizer.from_pretrained(__UpperCamelCase					)
							self.assertEqual(len(__UpperCamelCase					),    28_996					)
							_snake_case      			=			json.loads(old_tokenizer._tokenizer.to_str()					)
							self.assertNotIn("huggingface",    json_tokenizer["model"]["vocab"]					)





@require_tokenizers
class 	lowerCamelCase_				(			unittest.TestCase				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# See https://github.com/huggingface/transformers/pull/12550
							# and https://github.com/huggingface/tokenizers/issues/537
							_snake_case      			=			PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"					)
							_snake_case      			=			"The Matrix is a 1999 science fiction action film."

							with concurrent.futures.ThreadPoolExecutor() as executor:
									_snake_case      			=			[executor.submit(self.fetch,    __UpperCamelCase,    __UpperCamelCase					) for i in range(10					)]
									_snake_case      			=			[future.result() for future in futures]
									self.assertEqual(__UpperCamelCase,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10					)]					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase					):
							return tokenizer.encode(__UpperCamelCase,    truncation="longest_first",    padding="longest"					)





