






import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCAmelCase								=						get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')



@require_sentencepiece
@require_tokenizers
class   lowerCamelCase      (lowerCAmelCase_ ,  unittest.TestCase			):







		'''simple docstring'''



		_snake_case    :    Tuple       						=  GPTSwaTokenizer
		_snake_case    :    Tuple       						=  False
		_snake_case    :    Union[str, Any]       						=  True
		_snake_case    :    Tuple       						=  False



		def 					__UpperCAmelCase			(							self   )	->							Dict:
							super().setUp()

							# We have a SentencePiece fixture for testing
							UpperCAmelCase_    :      int            =						GPTSwaTokenizer(__snake_case				,		eos_token='<unk>'				,		bos_token='<unk>'				,		pad_token='<unk>'   )

							tokenizer.save_pretrained(self.tmpdirname   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Tuple:
							UpperCAmelCase_    :      List[Any]            =						'This is a test'
							UpperCAmelCase_    :      Optional[Any]            =						'This is a test'
							return input_text, output_text



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							UpperCAmelCase_    :      List[str]            =						'<s>'
							UpperCAmelCase_    :      Union[str, Any]            =						1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case   )				,		__snake_case   )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case   )				,		__snake_case   )



		def 					__UpperCAmelCase			(							self   )	->							Dict:
							UpperCAmelCase_    :      Tuple            =						list(self.get_tokenizer().get_vocab().keys()   )

							self.assertEqual(vocab_keys[0]				,		'<unk>'   )
							self.assertEqual(vocab_keys[1]				,		'<s>'   )
							self.assertEqual(vocab_keys[-1]				,		'j'   )
							self.assertEqual(len(__snake_case   )				,		2_0_0_0   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							self.assertEqual(self.get_tokenizer().vocab_size				,		2_0_0_0   )



		def 					__UpperCAmelCase			(							self   )	->							Any:
							UpperCAmelCase_    :      Optional[int]            =						GPTSwaTokenizer(__snake_case   )

							UpperCAmelCase_    :      Dict            =						tokenizer.tokenize('This is a test'   )
							self.assertListEqual(__snake_case				,		['▁This', '▁is', '▁a', '▁t', 'est']   )

							self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case   )				,		[4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2]   )

							UpperCAmelCase_    :      int            =						tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
							# fmt: off
							self.assertListEqual(
							    __snake_case				,		['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']				,		)
							# fmt: on

							UpperCAmelCase_    :      Optional[int]            =						tokenizer.convert_tokens_to_ids(__snake_case   )
							self.assertListEqual(
							    __snake_case				,		[2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0]				,		)

							UpperCAmelCase_    :      Dict            =						tokenizer.convert_ids_to_tokens(__snake_case   )
							# fmt: off
							self.assertListEqual(
							    __snake_case				,		['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']   )
							# fmt: on



		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							UpperCAmelCase_    :      Optional[Any]            =						GPTSwaTokenizer(__snake_case   )
							UpperCAmelCase_    :      Dict            =						['This is a test', 'I was born in 92000, and this is falsé.']
							UpperCAmelCase_    :      Optional[Any]            =						[
							    [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2],
							    [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0],
							]

							# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
							for text, expected_ids in zip(__snake_case				,		__snake_case   ):
												self.assertListEqual(tokenizer.encode_fast(__snake_case   )				,		__snake_case   )

							# Test that decode_fast returns the input text
							for text, token_ids in zip(__snake_case				,		__snake_case   ):
												self.assertEqual(tokenizer.decode_fast(__snake_case   )				,		__snake_case   )



		@slow
		def 					__UpperCAmelCase			(							self   )	->							List[str]:
							UpperCAmelCase_    :      Dict            =						[
							    '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
							    'Hey there, how are you doing this fine day?',
							    'This is a text with a trailing spaces followed by a dot     .',
							    'Häj sväjs lillebrör! =)',
							    'Det är inget fel på Mr. Cool',
							]

							# fmt: off
							UpperCAmelCase_    :      Optional[int]            =						{'input_ids': [[6_3_4_2_3, 5, 6_8_1_1, 1_4_9_5_4, 2_8_2, 8_1_6, 3_8_2_1, 6_3_4_6_6, 6_3_4_2_5, 6_3_4_6_2, 1_8, 6_3_9_7_8, 6_7_8, 3_0_1, 1_3_2_0, 6_3_4_2_3, 6_3_4_5_5, 6_3_4_5_8, 1_8, 6_3_9_8_2, 4_2_4_6, 3_9_4_0, 1_9_0_1, 4_7_7_8_9, 5_5_4_7, 1_8_9_9_4], [1_9_6_3_0, 1_1_0_0, 6_3_4_4_6, 1_3_4_2, 6_3_3, 5_4_4, 4_4_8_8, 5_9_3, 5_1_0_2, 2_4_1_6, 6_3_4_9_5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_6_5_2, 4_2_8, 2_6_8, 1_9_3_6, 5_1_5, 2_6_8, 5_8_5_9_3, 2_2_4_1_3, 9_1_0_6, 5_4_6, 2_6_8, 3_3_2_1_3, 6_3_9_7_9, 6_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5_1_3_0, 6_3_4_5_0, 9_2_4, 6_3_4_4_9, 2_2_4_9, 4_0_6_2, 1_5_5_8, 3_1_8, 6_3_5_0_4, 2_1_4_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_0_9, 3_7_7, 2_8_2_7, 2_5_5_9, 3_3_2, 6_5_7_5, 6_3_4_4_3, 2_6_8_0_1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
							# fmt: on
							self.tokenizer_integration_test_util(
							    expected_encoding=__snake_case				,		model_name='AI-Sweden/gpt-sw3-126m'				,		sequences=__snake_case				,		)









from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCAmelCase					:			Tuple  				=			logging.get_logger(__name__)

_lowerCAmelCase					:			Union[str, Any]  				=			{"ctrl": "https://huggingface.co/ctrl/resolve/main/config.json"}






class 	__magic_name__    (			lowerCAmelCase_      ):
			SCREAMING_SNAKE_CASE							   =						'ctrl'
			SCREAMING_SNAKE_CASE							   =						['past_key_values']
			SCREAMING_SNAKE_CASE							   =						{
			    'max_position_embeddings': 'n_positions',
			    'hidden_size': 'n_embd',
			    'num_attention_heads': 'n_head',
			    'num_hidden_layers': 'n_layer',
			}





			def __init__(		self					,  __snake_case=24_6534					,  __snake_case=256					,  __snake_case=1280					,  __snake_case=8192					,  __snake_case=48					,  __snake_case=16					,  __snake_case=0.1					,  __snake_case=0.1					,  __snake_case=1e-6					,  __snake_case=0.02					,  __snake_case=True					,  **__snake_case					,  )   -> Union[str, Any]:




						'''simple docstring'''
						__a        =vocab_size
						__a        =n_positions
						__a        =n_embd
						__a        =n_layer
						__a        =n_head
						__a        =dff
						__a        =resid_pdrop
						__a        =embd_pdrop
						__a        =layer_norm_epsilon
						__a        =initializer_range

						__a        =use_cache

						super().__init__(**__snake_case					)






