





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		UpperCamelCase__ (      lowerCAmelCase_							,      unittest.TestCase   ):



     '''simple docstring'''

     __snake_case    :	Union[str, Any]			=			CodeGenTokenizer
     __snake_case    :	Dict			=			CodeGenTokenizerFast
     __snake_case    :	str			=			True
     __snake_case    :	List[Any]			=			{'''add_prefix_space''': True}
     __snake_case    :	Dict			=			False






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple       )     ->		Optional[Any]:




           '''simple docstring'''


           super().setUp()

           # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
           SCREAMING_SNAKE_CASE		=						[
               """l""",
               """o""",
               """w""",
               """e""",
               """r""",
               """s""",
               """t""",
               """i""",
               """d""",
               """n""",
               """\u0120""",
               """\u0120l""",
               """\u0120n""",
               """\u0120lo""",
               """\u0120low""",
               """er""",
               """\u0120lowest""",
               """\u0120newer""",
               """\u0120wider""",
               """<unk>""",
               """<|endoftext|>""",
           ]
           SCREAMING_SNAKE_CASE		=						dict(zip(A__						,range(len(A__       )       )       )       )
           SCREAMING_SNAKE_CASE		=						["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
           SCREAMING_SNAKE_CASE		=						{"""unk_token""": """<unk>"""}

           SCREAMING_SNAKE_CASE		=						os.path.join(self.tmpdirname						,VOCAB_FILES_NAMES["""vocab_file"""]       )
           SCREAMING_SNAKE_CASE		=						os.path.join(self.tmpdirname						,VOCAB_FILES_NAMES["""merges_file"""]       )
           with open(self.vocab_file						,"""w"""						,encoding="""utf-8"""       ) as fp:
                 fp.write(json.dumps(A__       ) + """\n"""       )
           with open(self.merges_file						,"""w"""						,encoding="""utf-8"""       ) as fp:
                 fp.write("""\n""".join(A__       )       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[int]						,**lowerCamelCase__						:    List[str]       )     ->		Union[str, Any]:




           '''simple docstring'''


           kwargs.update(self.special_tokens_map       )
           return CodeGenTokenizer.from_pretrained(self.tmpdirname						,**A__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    List[Any]						,**lowerCamelCase__						:    Any       )     ->		Dict:




           '''simple docstring'''


           kwargs.update(self.special_tokens_map       )
           return CodeGenTokenizerFast.from_pretrained(self.tmpdirname						,**A__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[Any]						,lowerCamelCase__						:    Optional[Any]       )     ->		Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						"""lower newer"""
           SCREAMING_SNAKE_CASE		=						"""lower newer"""
           return input_text, output_text






     def 				SCREAMING_SNAKE_CASE__  (							self						:    str       )     ->		Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						CodeGenTokenizer(self.vocab_file						,self.merges_file						,**self.special_tokens_map       )
           SCREAMING_SNAKE_CASE		=						"""lower newer"""
           SCREAMING_SNAKE_CASE		=						["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize(A__						,add_prefix_space=A__       )
           self.assertListEqual(A__						,A__       )

           SCREAMING_SNAKE_CASE		=						tokens + [tokenizer.unk_token]
           SCREAMING_SNAKE_CASE		=						[14, 15, 10, 9, 3, 2, 15, 19]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(A__       )						,A__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[int]       )     ->		Union[str, Any]:




           '''simple docstring'''


           if not self.test_rust_tokenizer:
                 return

           SCREAMING_SNAKE_CASE		=						self.get_tokenizer()
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer(add_prefix_space=A__       )

           SCREAMING_SNAKE_CASE		=						"""lower newer"""

           # Testing tokenization
           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize(A__						,add_prefix_space=A__       )
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.tokenize(A__       )
           self.assertListEqual(A__						,A__       )

           # Testing conversion to ids without special tokens
           SCREAMING_SNAKE_CASE		=						tokenizer.encode(A__						,add_special_tokens=A__						,add_prefix_space=A__       )
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(A__						,add_special_tokens=A__       )
           self.assertListEqual(A__						,A__       )

           # Testing conversion to ids with special tokens
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer(add_prefix_space=A__       )
           SCREAMING_SNAKE_CASE		=						tokenizer.encode(A__						,add_prefix_space=A__       )
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(A__       )
           self.assertListEqual(A__						,A__       )

           # Testing the unknown token
           SCREAMING_SNAKE_CASE		=						tokens + [rust_tokenizer.unk_token]
           SCREAMING_SNAKE_CASE		=						[14, 15, 10, 9, 3, 2, 15, 19]
           self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A__       )						,A__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    List[Any]						,*lowerCamelCase__						:    Any						,**lowerCamelCase__						:    Optional[Any]       )     ->		List[str]:




           '''simple docstring'''


           pass






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Dict						,lowerCamelCase__						:    int=15       )     ->		List[str]:




           '''simple docstring'''


           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(A__						,**A__       )

                       # Simple input
                       SCREAMING_SNAKE_CASE		=						"""This is a simple input"""
                       SCREAMING_SNAKE_CASE		=						["""This is a simple input 1""", """This is a simple input 2"""]
                       SCREAMING_SNAKE_CASE		=						("""This is a simple input""", """This is a pair""")
                       SCREAMING_SNAKE_CASE		=						[
                           ("""This is a simple input 1""", """This is a simple input 2"""),
                           ("""This is a simple pair 1""", """This is a simple pair 2"""),
                       ]

                       # Simple input tests
                       self.assertRaises(A__						,tokenizer_r.encode						,A__						,max_length=A__						,padding="""max_length"""       )

                       # Simple input
                       self.assertRaises(A__						,tokenizer_r.encode_plus						,A__						,max_length=A__						,padding="""max_length"""       )

                       # Simple input
                       self.assertRaises(
                           A__						,tokenizer_r.batch_encode_plus						,A__						,max_length=A__						,padding="""max_length"""						,)

                       # Pair input
                       self.assertRaises(A__						,tokenizer_r.encode						,A__						,max_length=A__						,padding="""max_length"""       )

                       # Pair input
                       self.assertRaises(A__						,tokenizer_r.encode_plus						,A__						,max_length=A__						,padding="""max_length"""       )

                       # Pair input
                       self.assertRaises(
                           A__						,tokenizer_r.batch_encode_plus						,A__						,max_length=A__						,padding="""max_length"""						,)






     def 				SCREAMING_SNAKE_CASE__  (							self						:    str       )     ->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						CodeGenTokenizer.from_pretrained(self.tmpdirname						,pad_token="""<pad>"""       )

           # Simple input
           SCREAMING_SNAKE_CASE		=						"""This is a simple input"""
           SCREAMING_SNAKE_CASE		=						["""This is a simple input looooooooong""", """This is a simple input"""]
           SCREAMING_SNAKE_CASE		=						("""This is a simple input""", """This is a pair""")
           SCREAMING_SNAKE_CASE		=						[
               ("""This is a simple input loooooong""", """This is a simple input"""),
               ("""This is a simple pair loooooong""", """This is a simple pair"""),
           ]

           SCREAMING_SNAKE_CASE		=						tokenizer.pad_token_id

           SCREAMING_SNAKE_CASE		=						tokenizer(A__						,padding="""max_length"""						,max_length=30						,return_tensors="""np"""       )
           SCREAMING_SNAKE_CASE		=						tokenizer(A__						,padding=A__						,truncate=A__						,return_tensors="""np"""       )
           SCREAMING_SNAKE_CASE		=						tokenizer(*A__						,padding="""max_length"""						,max_length=60						,return_tensors="""np"""       )
           SCREAMING_SNAKE_CASE		=						tokenizer(A__						,padding=A__						,truncate=A__						,return_tensors="""np"""       )

           # s
           # test single string max_length padding
           self.assertEqual(out_s["""input_ids"""].shape[-1]						,30       )
           self.assertTrue(pad_token_id in out_s["""input_ids"""]       )
           self.assertTrue(0 in out_s["""attention_mask"""]       )

           # s2
           # test automatic padding
           self.assertEqual(out_sa["""input_ids"""].shape[-1]						,33       )
           # long slice doesn't have padding
           self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]       )
           self.assertFalse(0 in out_sa["""attention_mask"""][0]       )
           # short slice does have padding
           self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]       )
           self.assertTrue(0 in out_sa["""attention_mask"""][1]       )

           # p
           # test single pair max_length padding
           self.assertEqual(out_p["""input_ids"""].shape[-1]						,60       )
           self.assertTrue(pad_token_id in out_p["""input_ids"""]       )
           self.assertTrue(0 in out_p["""attention_mask"""]       )

           # p2
           # test automatic padding pair
           self.assertEqual(out_pa["""input_ids"""].shape[-1]						,52       )
           # long slice pair doesn't have padding
           self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]       )
           self.assertFalse(0 in out_pa["""attention_mask"""][0]       )
           # short slice pair does have padding
           self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]       )
           self.assertTrue(0 in out_pa["""attention_mask"""][1]       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[Any]       )     ->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						"""$$$"""
           SCREAMING_SNAKE_CASE		=						CodeGenTokenizer.from_pretrained(self.tmpdirname						,bos_token=A__						,add_bos_token=A__       )

           SCREAMING_SNAKE_CASE		=						"""This is a simple input"""
           SCREAMING_SNAKE_CASE		=						["""This is a simple input 1""", """This is a simple input 2"""]

           SCREAMING_SNAKE_CASE		=						tokenizer.bos_token_id

           SCREAMING_SNAKE_CASE		=						tokenizer(A__       )
           SCREAMING_SNAKE_CASE		=						tokenizer(A__       )

           self.assertEqual(out_s.input_ids[0]						,A__       )
           self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

           SCREAMING_SNAKE_CASE		=						tokenizer.decode(out_s.input_ids       )
           SCREAMING_SNAKE_CASE		=						tokenizer.batch_decode(out_sa.input_ids       )

           self.assertEqual(decode_s.split()[0]						,A__       )
           self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )






     @slow
     def 				SCREAMING_SNAKE_CASE__  (							self						:    str       )     ->		Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""       )

           SCREAMING_SNAKE_CASE		=						"""\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
           SCREAMING_SNAKE_CASE		=						"""\nif len_a > len_b:      result = a\nelse:      result = b"""

           SCREAMING_SNAKE_CASE		=						tokenizer.encode(A__       )
           SCREAMING_SNAKE_CASE		=						["""^#""", re.escape("""<|endoftext|>"""       ), """^'''""", """^\"\"\"""", """\n\n\n"""]
           SCREAMING_SNAKE_CASE		=						tokenizer.decode(A__						,truncate_before_pattern=A__       )
           self.assertEqual(A__						,A__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple       )     ->		Any:




           '''simple docstring'''


           pass





import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin



class     _a						(__magic_name__		,   __magic_name__		,   __magic_name__				):


					'''simple docstring'''






					@register_to_config
					def __init__( self    ,       A__    ,       A__    ,       A__    ,       A__    ,       A__    ,       A__    ,       A__    ,       A__    ,       A__    ,       A__ = False    ,       ):
						super().__init__()

						A__							:    List[str]        =		nn.Embedding(A__    ,       A__			)

						A__							:    Any        =		nn.Embedding(A__    ,       A__			)
						A__							:    Optional[int]        =		False

						A__							:    Optional[Any]        =		nn.Dropout(p=A__			)

						A__							:    int        =		TaConfig(
						    vocab_size=A__    ,       d_model=A__    ,       num_heads=A__    ,       d_kv=A__    ,       d_ff=A__    ,       dropout_rate=A__    ,       feed_forward_proj=A__    ,       is_decoder=A__    ,       is_encoder_decoder=A__    ,       )

						A__							:    Dict        =		nn.ModuleList()
						for lyr_num in range(A__			):
							A__							:    str        =		TaBlock(A__			)
							self.encoders.append(A__			)

						A__							:    Tuple        =		TaLayerNorm(A__			)
						A__							:    str        =		nn.Dropout(p=A__			)






					def   __A			( self    ,       A__    ,       A__			):
						A__							:    List[str]        =		self.token_embedder(A__			)

						A__							:    Optional[Any]        =		encoder_input_tokens.shape[1]
						A__							:    Optional[int]        =		torch.arange(A__    ,       device=encoder_input_tokens.device			)
						x += self.position_encoding(A__			)

						A__							:    Optional[Any]        =		self.dropout_pre(A__			)

						# inverted the attention mask
						A__							:    Tuple        =		encoder_input_tokens.size()
						A__							:    Union[str, Any]        =		self.get_extended_attention_mask(A__    ,       A__			)

						for lyr in self.encoders:
							A__							:    List[str]        =		lyr(A__    ,       A__			)[0]
						A__							:    List[Any]        =		self.layer_norm(A__			)

						return self.dropout_post(A__			), encoder_inputs_mask



