




import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		__lowerCAmelCase	(	__magic_name__      ,       unittest.TestCase ):
							UpperCamelCase__					     =  CodeGenTokenizer
							UpperCamelCase__					     =  CodeGenTokenizerFast
							UpperCamelCase__					     =  True
							UpperCamelCase__					     =  {'''add_prefix_space''': True}
							UpperCamelCase__					     =  False
							def       lowerCamelCase__							(						self						:List[str]				):







								'''simple docstring'''







								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								a										=							[
								    """l""",
								    """o""",
								    """w""",
								    """e""",
								    """r""",
								    """s""",
								    """t""",
								    """i""",
								    """d""",
								    """n""",
								    """\u0120""",
								    """\u0120l""",
								    """\u0120n""",
								    """\u0120lo""",
								    """\u0120low""",
								    """er""",
								    """\u0120lowest""",
								    """\u0120newer""",
								    """\u0120wider""",
								    """<unk>""",
								    """<|endoftext|>""",
								]
								a										=							dict(zip(__magic_name__		,      range(len(__magic_name__				)				)				)				)
								a										=							["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
								a										=							{"""unk_token""": """<unk>"""}

								a										=							os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["""vocab_file"""]				)
								a										=							os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["""merges_file"""]				)
								with open(self.vocab_file		,      """w"""		,      encoding="""utf-8"""				) as fp:
									fp.write(json.dumps(__magic_name__				) + """\n"""				)
								with open(self.merges_file		,      """w"""		,      encoding="""utf-8"""				) as fp:
									fp.write("""\n""".join(__magic_name__				)				)
							def       lowerCamelCase__							(						self						:str		,      **__magic_name__						:int				):







								'''simple docstring'''







								kwargs.update(self.special_tokens_map				)
								return CodeGenTokenizer.from_pretrained(self.tmpdirname		,      **__magic_name__				)
							def       lowerCamelCase__							(						self						:str		,      **__magic_name__						:Optional[Any]				):







								'''simple docstring'''







								kwargs.update(self.special_tokens_map				)
								return CodeGenTokenizerFast.from_pretrained(self.tmpdirname		,      **__magic_name__				)
							def       lowerCamelCase__							(						self						:Dict		,      __magic_name__						:Any				):







								'''simple docstring'''







								a										=							"""lower newer"""
								a										=							"""lower newer"""
								return input_text, output_text
							def       lowerCamelCase__							(						self						:Union[str, Any]				):







								'''simple docstring'''







								a										=							CodeGenTokenizer(self.vocab_file		,      self.merges_file		,      **self.special_tokens_map				)
								a										=							"""lower newer"""
								a										=							["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
								a										=							tokenizer.tokenize(__magic_name__		,      add_prefix_space=__magic_name__				)
								self.assertListEqual(__magic_name__		,      __magic_name__				)

								a										=							tokens + [tokenizer.unk_token]
								a										=							[14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__				)		,      __magic_name__				)
							def       lowerCamelCase__							(						self						:int				):







								'''simple docstring'''







								if not self.test_rust_tokenizer:
									return

								a										=							self.get_tokenizer()
								a										=							self.get_rust_tokenizer(add_prefix_space=__magic_name__				)

								a										=							"""lower newer"""

								# Testing tokenization
								a										=							tokenizer.tokenize(__magic_name__		,      add_prefix_space=__magic_name__				)
								a										=							rust_tokenizer.tokenize(__magic_name__				)
								self.assertListEqual(__magic_name__		,      __magic_name__				)

								# Testing conversion to ids without special tokens
								a										=							tokenizer.encode(__magic_name__		,      add_special_tokens=__magic_name__		,      add_prefix_space=__magic_name__				)
								a										=							rust_tokenizer.encode(__magic_name__		,      add_special_tokens=__magic_name__				)
								self.assertListEqual(__magic_name__		,      __magic_name__				)

								# Testing conversion to ids with special tokens
								a										=							self.get_rust_tokenizer(add_prefix_space=__magic_name__				)
								a										=							tokenizer.encode(__magic_name__		,      add_prefix_space=__magic_name__				)
								a										=							rust_tokenizer.encode(__magic_name__				)
								self.assertListEqual(__magic_name__		,      __magic_name__				)

								# Testing the unknown token
								a										=							tokens + [rust_tokenizer.unk_token]
								a										=							[14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__magic_name__				)		,      __magic_name__				)
							def       lowerCamelCase__							(						self						:Dict		,      *__magic_name__						:Union[str, Any]		,      **__magic_name__						:int				):







								'''simple docstring'''







								pass
							def       lowerCamelCase__							(						self						:Union[str, Any]		,      __magic_name__						:Tuple=15				):







								'''simple docstring'''







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
										a										=							self.rust_tokenizer_class.from_pretrained(__magic_name__		,      **__magic_name__				)

										# Simple input
										a										=							"""This is a simple input"""
										a										=							["""This is a simple input 1""", """This is a simple input 2"""]
										a										=							("""This is a simple input""", """This is a pair""")
										a										=							[
										    ("""This is a simple input 1""", """This is a simple input 2"""),
										    ("""This is a simple pair 1""", """This is a simple pair 2"""),
										]

										# Simple input tests
										self.assertRaises(__magic_name__		,      tokenizer_r.encode		,      __magic_name__		,      max_length=__magic_name__		,      padding="""max_length"""				)

										# Simple input
										self.assertRaises(__magic_name__		,      tokenizer_r.encode_plus		,      __magic_name__		,      max_length=__magic_name__		,      padding="""max_length"""				)

										# Simple input
										self.assertRaises(
										    __magic_name__		,      tokenizer_r.batch_encode_plus		,      __magic_name__		,      max_length=__magic_name__		,      padding="""max_length"""		,      )

										# Pair input
										self.assertRaises(__magic_name__		,      tokenizer_r.encode		,      __magic_name__		,      max_length=__magic_name__		,      padding="""max_length"""				)

										# Pair input
										self.assertRaises(__magic_name__		,      tokenizer_r.encode_plus		,      __magic_name__		,      max_length=__magic_name__		,      padding="""max_length"""				)

										# Pair input
										self.assertRaises(
										    __magic_name__		,      tokenizer_r.batch_encode_plus		,      __magic_name__		,      max_length=__magic_name__		,      padding="""max_length"""		,      )
							def       lowerCamelCase__							(						self						:List[str]				):







								'''simple docstring'''







								a										=							CodeGenTokenizer.from_pretrained(self.tmpdirname		,      pad_token="""<pad>"""				)

								# Simple input
								a										=							"""This is a simple input"""
								a										=							["""This is a simple input looooooooong""", """This is a simple input"""]
								a										=							("""This is a simple input""", """This is a pair""")
								a										=							[
								    ("""This is a simple input loooooong""", """This is a simple input"""),
								    ("""This is a simple pair loooooong""", """This is a simple pair"""),
								]

								a										=							tokenizer.pad_token_id

								a										=							tokenizer(__magic_name__		,      padding="""max_length"""		,      max_length=30		,      return_tensors="""np"""				)
								a										=							tokenizer(__magic_name__		,      padding=__magic_name__		,      truncate=__magic_name__		,      return_tensors="""np"""				)
								a										=							tokenizer(*__magic_name__		,      padding="""max_length"""		,      max_length=60		,      return_tensors="""np"""				)
								a										=							tokenizer(__magic_name__		,      padding=__magic_name__		,      truncate=__magic_name__		,      return_tensors="""np"""				)

								# s
								# test single string max_length padding
								self.assertEqual(out_s["""input_ids"""].shape[-1]		,      30				)
								self.assertTrue(pad_token_id in out_s["""input_ids"""]				)
								self.assertTrue(0 in out_s["""attention_mask"""]				)

								# s2
								# test automatic padding
								self.assertEqual(out_sa["""input_ids"""].shape[-1]		,      33				)
								# long slice doesn't have padding
								self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]				)
								self.assertFalse(0 in out_sa["""attention_mask"""][0]				)
								# short slice does have padding
								self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]				)
								self.assertTrue(0 in out_sa["""attention_mask"""][1]				)

								# p
								# test single pair max_length padding
								self.assertEqual(out_p["""input_ids"""].shape[-1]		,      60				)
								self.assertTrue(pad_token_id in out_p["""input_ids"""]				)
								self.assertTrue(0 in out_p["""attention_mask"""]				)

								# p2
								# test automatic padding pair
								self.assertEqual(out_pa["""input_ids"""].shape[-1]		,      52				)
								# long slice pair doesn't have padding
								self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]				)
								self.assertFalse(0 in out_pa["""attention_mask"""][0]				)
								# short slice pair does have padding
								self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]				)
								self.assertTrue(0 in out_pa["""attention_mask"""][1]				)
							def       lowerCamelCase__							(						self						:List[Any]				):







								'''simple docstring'''







								a										=							"""$$$"""
								a										=							CodeGenTokenizer.from_pretrained(self.tmpdirname		,      bos_token=__magic_name__		,      add_bos_token=__magic_name__				)

								a										=							"""This is a simple input"""
								a										=							["""This is a simple input 1""", """This is a simple input 2"""]

								a										=							tokenizer.bos_token_id

								a										=							tokenizer(__magic_name__				)
								a										=							tokenizer(__magic_name__				)

								self.assertEqual(out_s.input_ids[0]		,      __magic_name__				)
								self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids				)				)

								a										=							tokenizer.decode(out_s.input_ids				)
								a										=							tokenizer.batch_decode(out_sa.input_ids				)

								self.assertEqual(decode_s.split()[0]		,      __magic_name__				)
								self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa				)				)
							@slow
							def       lowerCamelCase__							(						self						:List[str]				):







								'''simple docstring'''







								a										=							CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""				)

								a										=							"""\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
								a										=							"""\nif len_a > len_b:      result = a\nelse:      result = b"""

								a										=							tokenizer.encode(__magic_name__				)
								a										=							["""^#""", re.escape("""<|endoftext|>"""				), """^'''""", """^\"\"\"""", """\n\n\n"""]
								a										=							tokenizer.decode(__magic_name__		,      truncate_before_pattern=__magic_name__				)
								self.assertEqual(__magic_name__		,      __magic_name__				)





							def       lowerCamelCase__							(						self						:List[str]				):







								'''simple docstring'''







								pass







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		__lowerCAmelCase	(	__magic_name__      ,       unittest.TestCase ):
							UpperCamelCase__					     =  CLIPTokenizer
							UpperCamelCase__					     =  CLIPTokenizerFast
							UpperCamelCase__					     =  True
							UpperCamelCase__					     =  {}
							UpperCamelCase__					     =  False
							def       lowerCamelCase__							(						self						:str				):







								'''simple docstring'''







								super().setUp()

								# fmt: off
								a										=							["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
								# fmt: on
								a										=							dict(zip(__magic_name__		,      range(len(__magic_name__				)				)				)				)
								a										=							["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
								a										=							{"""unk_token""": """<unk>"""}

								a										=							os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["""vocab_file"""]				)
								a										=							os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["""merges_file"""]				)
								with open(self.vocab_file		,      """w"""		,      encoding="""utf-8"""				) as fp:
									fp.write(json.dumps(__magic_name__				) + """\n"""				)
								with open(self.merges_file		,      """w"""		,      encoding="""utf-8"""				) as fp:
									fp.write("""\n""".join(__magic_name__				)				)
							def       lowerCamelCase__							(						self						:Optional[Any]		,      **__magic_name__						:Union[str, Any]				):







								'''simple docstring'''







								kwargs.update(self.special_tokens_map				)
								return CLIPTokenizer.from_pretrained(self.tmpdirname		,      **__magic_name__				)
							def       lowerCamelCase__							(						self						:int		,      **__magic_name__						:Optional[Any]				):







								'''simple docstring'''







								kwargs.update(self.special_tokens_map				)
								return CLIPTokenizerFast.from_pretrained(self.tmpdirname		,      **__magic_name__				)
							def       lowerCamelCase__							(						self						:int		,      __magic_name__						:List[str]				):







								'''simple docstring'''







								a										=							"""lower newer"""
								a										=							"""lower newer"""
								return input_text, output_text
							def       lowerCamelCase__							(						self						:int				):







								'''simple docstring'''







								a										=							CLIPTokenizer(self.vocab_file		,      self.merges_file		,      **self.special_tokens_map				)
								a										=							"""lower newer"""
								a										=							["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
								a										=							tokenizer.tokenize(__magic_name__				)
								self.assertListEqual(__magic_name__		,      __magic_name__				)

								a										=							tokens + [tokenizer.unk_token]
								a										=							[10, 2, 16, 9, 3, 2, 16, 20]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__				)		,      __magic_name__				)
							@require_ftfy
							def       lowerCamelCase__							(						self						:List[Any]				):







								'''simple docstring'''







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
										a										=							self.tokenizer_class.from_pretrained(__magic_name__		,      **__magic_name__				)
										a										=							self.rust_tokenizer_class.from_pretrained(__magic_name__		,      **__magic_name__				)

										a										=							"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
										a										=							tokenizer_s.tokenize(__magic_name__				)
										a										=							tokenizer_r.tokenize(__magic_name__				)

										self.assertListEqual(__magic_name__		,      __magic_name__				)

										# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
										# with Tilde) encoded in 2 different ways
										a										=							"""xa\u0303y""" + """ """ + """x\xe3y"""
										a										=							tokenizer_s.tokenize(__magic_name__				)
										a										=							tokenizer_r.tokenize(__magic_name__				)

										self.assertListEqual(__magic_name__		,      __magic_name__				)

										# Test that the tokenization is identical on unicode of space type
										a										=							[
										    """\u0009""",  # (horizontal tab, '\t')
										    """\u000B""",  # (vertical tab)
										    """\u000C""",  # (form feed)
										    """\u0020""",  # (space, ' ')
										    """\u200E""",  # (left-to-right mark):w
										    """\u200F""",  # (right-to-left mark)
										]
										for unicode_seq in spaces_unicodes:
											a										=							tokenizer_s.tokenize(__magic_name__				)
											a										=							tokenizer_r.tokenize(__magic_name__				)

											self.assertListEqual(__magic_name__		,      __magic_name__				)

										# Test that the tokenization is identical on unicode of line break type
										a										=							[
										    """\u000A""",  # (line feed, '\n')
										    """\r\n""",  # (carriage return and line feed, '\r\n')
										    """\u000D""",  # (carriage return, '\r')
										    """\r""",  # (carriage return, '\r')
										    """\u000D""",  # (carriage return, '\r')
										    """\u2028""",  # (line separator)
										    """\u2029""",  # (paragraph separator)
										    # "\u0085", # (next line)
										]

										# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
										# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
										# space (and thus into an empty list).

										for unicode_seq in line_break_unicodes:
											a										=							tokenizer_s.tokenize(__magic_name__				)
											a										=							tokenizer_r.tokenize(__magic_name__				)

											self.assertListEqual(__magic_name__		,      __magic_name__				)
							def       lowerCamelCase__							(						self						:int				):







								'''simple docstring'''







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
										a										=							"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
										a										=							F'{text_of_1_token} {text_of_1_token}'

										a										=							self.rust_tokenizer_class.from_pretrained(
										    __magic_name__		,      use_fast=__magic_name__		,      )
										a										=							tokenizer_r(__magic_name__		,      return_offsets_mapping=__magic_name__		,      add_special_tokens=__magic_name__				)
										self.assertEqual(encoding.offset_mapping[0]		,      (0, len(__magic_name__				))				)
										self.assertEqual(
										    encoding.offset_mapping[1]		,      (len(__magic_name__				) + 1, len(__magic_name__				) + 1 + len(__magic_name__				))		,      )

										a										=							F' {text}'

										a										=							self.rust_tokenizer_class.from_pretrained(
										    __magic_name__		,      use_fast=__magic_name__		,      )
										a										=							tokenizer_r(__magic_name__		,      return_offsets_mapping=__magic_name__		,      add_special_tokens=__magic_name__				)
										self.assertEqual(encoding.offset_mapping[0]		,      (1, 1 + len(__magic_name__				))				)
										self.assertEqual(
										    encoding.offset_mapping[1]		,      (1 + len(__magic_name__				) + 1, 1 + len(__magic_name__				) + 1 + len(__magic_name__				))		,      )
							def       lowerCamelCase__							(						self						:List[str]				):







								'''simple docstring'''







								with self.assertRaises(__magic_name__				) as context:
									self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""				)

								self.assertTrue(
								    context.exception.args[0].startswith(
								        """The `backend_tokenizer` provided does not match the expected format."""				)				)
							@require_ftfy
							def       lowerCamelCase__							(						self						:Tuple				):







								'''simple docstring'''







								super().test_tokenization_python_rust_equals()





							def       lowerCamelCase__							(						self						:Optional[int]				):







								'''simple docstring'''







								pass

