"""simple docstring"""




import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 						__A							( __lowerCamelCase    ,	unittest.TestCase		):
     _UpperCamelCase							:			Union[str, Any]							=		CodeGenTokenizer
     _UpperCamelCase							:			Optional[Any]							=		CodeGenTokenizerFast
     _UpperCamelCase							:			Union[str, Any]							=		True
     _UpperCamelCase							:			Union[str, Any]							=		{"""add_prefix_space""": True}
     _UpperCamelCase							:			Any							=		False






     def     __A  (		self  ):
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       _lowerCAmelCase						:       List[str]  			=     [
           '''l''',
           '''o''',
           '''w''',
           '''e''',
           '''r''',
           '''s''',
           '''t''',
           '''i''',
           '''d''',
           '''n''',
           '''\u0120''',
           '''\u0120l''',
           '''\u0120n''',
           '''\u0120lo''',
           '''\u0120low''',
           '''er''',
           '''\u0120lowest''',
           '''\u0120newer''',
           '''\u0120wider''',
           '''<unk>''',
           '''<|endoftext|>''',
       ]
       _lowerCAmelCase						:       str  			=     dict(zip(__lowercase				,     range(len(__lowercase  )  )  )  )
       _lowerCAmelCase						:       Tuple  			=     ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
       _lowerCAmelCase						:       Any  			=     {'''unk_token''': '''<unk>'''}

       _lowerCAmelCase						:       List[str]  			=     os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""vocab_file"""]  )
       _lowerCAmelCase						:       Optional[Any]  			=     os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""merges_file"""]  )
       with open(self.vocab_file				,     """w"""				,     encoding="""utf-8"""  ) as fp:
         fp.write(json.dumps(__lowercase  ) + """\n"""  )
       with open(self.merges_file				,     """w"""				,     encoding="""utf-8"""  ) as fp:
         fp.write("""\n""".join(__lowercase  )  )






     def     __A  (		self				,     **a__  ):
       kwargs.update(self.special_tokens_map  )
       return CodeGenTokenizer.from_pretrained(self.tmpdirname				,     **__lowercase  )






     def     __A  (		self				,     **a__  ):
       kwargs.update(self.special_tokens_map  )
       return CodeGenTokenizerFast.from_pretrained(self.tmpdirname				,     **__lowercase  )






     def     __A  (		self				,     a__  ):
       _lowerCAmelCase						:       Union[str, Any]  			=     '''lower newer'''
       _lowerCAmelCase						:       Tuple  			=     '''lower newer'''
       return input_text, output_text






     def     __A  (		self  ):
       _lowerCAmelCase						:       str  			=     CodeGenTokenizer(self.vocab_file				,     self.merges_file				,     **self.special_tokens_map  )
       _lowerCAmelCase						:       str  			=     '''lower newer'''
       _lowerCAmelCase						:       Dict  			=     ['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
       _lowerCAmelCase						:       Union[str, Any]  			=     tokenizer.tokenize(__lowercase				,     add_prefix_space=__lowercase  )
       self.assertListEqual(__lowercase				,     __lowercase  )

       _lowerCAmelCase						:       List[str]  			=     tokens + [tokenizer.unk_token]
       _lowerCAmelCase						:       int  			=     [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase  )				,     __lowercase  )






     def     __A  (		self  ):
       if not self.test_rust_tokenizer:
         return

       _lowerCAmelCase						:       List[Any]  			=     self.get_tokenizer()
       _lowerCAmelCase						:       List[Any]  			=     self.get_rust_tokenizer(add_prefix_space=__lowercase  )

       _lowerCAmelCase						:       Tuple  			=     '''lower newer'''

       # Testing tokenization
       _lowerCAmelCase						:       int  			=     tokenizer.tokenize(__lowercase				,     add_prefix_space=__lowercase  )
       _lowerCAmelCase						:       str  			=     rust_tokenizer.tokenize(__lowercase  )
       self.assertListEqual(__lowercase				,     __lowercase  )

       # Testing conversion to ids without special tokens
       _lowerCAmelCase						:       Tuple  			=     tokenizer.encode(__lowercase				,     add_special_tokens=__lowercase				,     add_prefix_space=__lowercase  )
       _lowerCAmelCase						:       Optional[int]  			=     rust_tokenizer.encode(__lowercase				,     add_special_tokens=__lowercase  )
       self.assertListEqual(__lowercase				,     __lowercase  )

       # Testing conversion to ids with special tokens
       _lowerCAmelCase						:       Any  			=     self.get_rust_tokenizer(add_prefix_space=__lowercase  )
       _lowerCAmelCase						:       Tuple  			=     tokenizer.encode(__lowercase				,     add_prefix_space=__lowercase  )
       _lowerCAmelCase						:       List[Any]  			=     rust_tokenizer.encode(__lowercase  )
       self.assertListEqual(__lowercase				,     __lowercase  )

       # Testing the unknown token
       _lowerCAmelCase						:       Tuple  			=     tokens + [rust_tokenizer.unk_token]
       _lowerCAmelCase						:       List[str]  			=     [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowercase  )				,     __lowercase  )






     def     __A  (		self				,     *a__				,     **a__  ):
       # It's very difficult to mix/test pretokenization with byte-level
       # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
       pass






     def     __A  (		self				,     a__=15  ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"  ):
           _lowerCAmelCase						:       List[str]  			=     self.rust_tokenizer_class.from_pretrained(__lowercase				,     **__lowercase  )

           # Simple input
           _lowerCAmelCase						:       str  			=     '''This is a simple input'''
           _lowerCAmelCase						:       Any  			=     ['''This is a simple input 1''', '''This is a simple input 2''']
           _lowerCAmelCase						:       Any  			=     ('''This is a simple input''', '''This is a pair''')
           _lowerCAmelCase						:       List[str]  			=     [
               ('''This is a simple input 1''', '''This is a simple input 2'''),
               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
           ]

           # Simple input tests
           self.assertRaises(__lowercase				,     tokenizer_r.encode				,     __lowercase				,     max_length=__lowercase				,     padding="""max_length"""  )

           # Simple input
           self.assertRaises(__lowercase				,     tokenizer_r.encode_plus				,     __lowercase				,     max_length=__lowercase				,     padding="""max_length"""  )

           # Simple input
           self.assertRaises(
               __lowercase				,     tokenizer_r.batch_encode_plus				,     __lowercase				,     max_length=__lowercase				,     padding="""max_length"""				,     )

           # Pair input
           self.assertRaises(__lowercase				,     tokenizer_r.encode				,     __lowercase				,     max_length=__lowercase				,     padding="""max_length"""  )

           # Pair input
           self.assertRaises(__lowercase				,     tokenizer_r.encode_plus				,     __lowercase				,     max_length=__lowercase				,     padding="""max_length"""  )

           # Pair input
           self.assertRaises(
               __lowercase				,     tokenizer_r.batch_encode_plus				,     __lowercase				,     max_length=__lowercase				,     padding="""max_length"""				,     )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Union[str, Any]  			=     CodeGenTokenizer.from_pretrained(self.tmpdirname				,     pad_token="""<pad>"""  )

       # Simple input
       _lowerCAmelCase						:       List[Any]  			=     '''This is a simple input'''
       _lowerCAmelCase						:       Union[str, Any]  			=     ['''This is a simple input looooooooong''', '''This is a simple input''']
       _lowerCAmelCase						:       Dict  			=     ('''This is a simple input''', '''This is a pair''')
       _lowerCAmelCase						:       int  			=     [
           ('''This is a simple input loooooong''', '''This is a simple input'''),
           ('''This is a simple pair loooooong''', '''This is a simple pair'''),
       ]

       _lowerCAmelCase						:       Optional[int]  			=     tokenizer.pad_token_id

       _lowerCAmelCase						:       Optional[int]  			=     tokenizer(__lowercase				,     padding="""max_length"""				,     max_length=30				,     return_tensors="""np"""  )
       _lowerCAmelCase						:       Any  			=     tokenizer(__lowercase				,     padding=__lowercase				,     truncate=__lowercase				,     return_tensors="""np"""  )
       _lowerCAmelCase						:       Any  			=     tokenizer(*__lowercase				,     padding="""max_length"""				,     max_length=60				,     return_tensors="""np"""  )
       _lowerCAmelCase						:       str  			=     tokenizer(__lowercase				,     padding=__lowercase				,     truncate=__lowercase				,     return_tensors="""np"""  )

       # s
       # test single string max_length padding
       self.assertEqual(out_s["""input_ids"""].shape[-1]				,     30  )
       self.assertTrue(pad_token_id in out_s["""input_ids"""]  )
       self.assertTrue(0 in out_s["""attention_mask"""]  )

       # s2
       # test automatic padding
       self.assertEqual(out_sa["""input_ids"""].shape[-1]				,     33  )
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]  )
       self.assertFalse(0 in out_sa["""attention_mask"""][0]  )
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]  )
       self.assertTrue(0 in out_sa["""attention_mask"""][1]  )

       # p
       # test single pair max_length padding
       self.assertEqual(out_p["""input_ids"""].shape[-1]				,     60  )
       self.assertTrue(pad_token_id in out_p["""input_ids"""]  )
       self.assertTrue(0 in out_p["""attention_mask"""]  )

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa["""input_ids"""].shape[-1]				,     52  )
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]  )
       self.assertFalse(0 in out_pa["""attention_mask"""][0]  )
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]  )
       self.assertTrue(0 in out_pa["""attention_mask"""][1]  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       List[Any]  			=     '''$$$'''
       _lowerCAmelCase						:       List[str]  			=     CodeGenTokenizer.from_pretrained(self.tmpdirname				,     bos_token=__lowercase				,     add_bos_token=__lowercase  )

       _lowerCAmelCase						:       str  			=     '''This is a simple input'''
       _lowerCAmelCase						:       List[str]  			=     ['''This is a simple input 1''', '''This is a simple input 2''']

       _lowerCAmelCase						:       List[str]  			=     tokenizer.bos_token_id

       _lowerCAmelCase						:       Optional[int]  			=     tokenizer(__lowercase  )
       _lowerCAmelCase						:       Union[str, Any]  			=     tokenizer(__lowercase  )

       self.assertEqual(out_s.input_ids[0]				,     __lowercase  )
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids  )  )

       _lowerCAmelCase						:       Union[str, Any]  			=     tokenizer.decode(out_s.input_ids  )
       _lowerCAmelCase						:       List[str]  			=     tokenizer.batch_decode(out_sa.input_ids  )

       self.assertEqual(decode_s.split()[0]				,     __lowercase  )
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa  )  )






     @slow
     def     __A  (		self  ):
       _lowerCAmelCase						:       str  			=     CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""  )

       _lowerCAmelCase						:       Dict  			=     '''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
       _lowerCAmelCase						:       Dict  			=     '''\nif len_a > len_b:      result = a\nelse:      result = b'''

       _lowerCAmelCase						:       Any  			=     tokenizer.encode(__lowercase  )
       _lowerCAmelCase						:       Dict  			=     ['''^#''', re.escape("""<|endoftext|>"""  ), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
       _lowerCAmelCase						:       Optional[Any]  			=     tokenizer.decode(__lowercase				,     truncate_before_pattern=__lowercase  )
       self.assertEqual(__lowercase				,     __lowercase  )






     def     __A  (		self  ):
       pass









A__								=						[4, 1, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
A__								=						[3, 7, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
A__								=						{
    0: '''Sunday''',
    1: '''Monday''',
    2: '''Tuesday''',
    3: '''Wednesday''',
    4: '''Thursday''',
    5: '''Friday''',
    6: '''Saturday''',
}



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase      )   ->					str:


				"""simple docstring"""

				assert len(str(__lowerCAmelCase      )      ) > 2, "year should be in YYYY format"
				assert 1 <= month <= 12, "month should be between 1 to 12"
				assert 1 <= day <= 31, "day should be between 1 to 31"

				# Doomsday algorithm:
				snake_case__	:      Optional[int]     		=	year // 100
				snake_case__	:      List[str]     		=	(5 * (century % 4) + 2) % 7
				snake_case__	:      Dict     		=	year % 100
				snake_case__	:      Union[str, Any]     		=	centurian % 12
				snake_case__	:      List[Any]     		=	(
				    (centurian // 12) + centurian_m + (centurian_m // 4) + century_anchor
				) % 7
				snake_case__	:      List[str]     		=	(
				    DOOMSDAY_NOT_LEAP[month - 1]
				    if (year % 4 != 0) or (centurian == 0 and (year % 400) == 0)
				    else DOOMSDAY_LEAP[month - 1]
				)
				snake_case__	:      List[str]     		=	(dooms_day + day - day_anchor) % 7
				return WEEK_DAY_NAMES[week_day]


if __name__ == "__main__":
				import doctest

				doctest.testmod()

