





'''simple docstring'''






import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  UpperCAmelCase     (			snake_case_							,		unittest.TestCase   ):
      _lowercase:			List[Any]						     =   LEDTokenizer
      _lowercase:			Dict						     =   LEDTokenizerFast
      _lowercase:			List[str]						     =   True




      def 				lowercase__    (			self			: Optional[int]  )				->      str:
            super().setUp()
            _lowerCAmelCase							     =							[
                """l""",
                """o""",
                """w""",
                """e""",
                """r""",
                """s""",
                """t""",
                """i""",
                """d""",
                """n""",
                """\u0120""",
                """\u0120l""",
                """\u0120n""",
                """\u0120lo""",
                """\u0120low""",
                """er""",
                """\u0120lowest""",
                """\u0120newer""",
                """\u0120wider""",
                """<unk>""",
            ]
            _lowerCAmelCase							     =							dict(zip(__snake_case			,			range(len(__snake_case  )  )  )  )
            _lowerCAmelCase							     =							["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
            _lowerCAmelCase							     =							{"""unk_token""": """<unk>"""}

            _lowerCAmelCase							     =							os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES["""vocab_file"""]  )
            _lowerCAmelCase							     =							os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES["""merges_file"""]  )
            with open(self.vocab_file			,			"""w"""			,			encoding="""utf-8"""  ) as fp:
                  fp.write(json.dumps(__snake_case  ) + """\n"""  )
            with open(self.merges_file			,			"""w"""			,			encoding="""utf-8"""  ) as fp:
                  fp.write("""\n""".join(__snake_case  )  )




      def 				lowercase__    (			self			: List[Any]			,			**__snake_case			: str  )				->      Union[str, Any]:
            kwargs.update(self.special_tokens_map  )
            return self.tokenizer_class.from_pretrained(self.tmpdirname			,			**__snake_case  )




      def 				lowercase__    (			self			: int			,			**__snake_case			: Optional[int]  )				->      List[str]:
            kwargs.update(self.special_tokens_map  )
            return self.rust_tokenizer_class.from_pretrained(self.tmpdirname			,			**__snake_case  )




      def 				lowercase__    (			self			: List[Any]			,			__snake_case			: Optional[Any]  )				->      Tuple:
            return "lower newer", "lower newer"




      @cached_property
      def 				lowercase__    (			self			: Optional[int]  )				->      Optional[Any]:
            return LEDTokenizer.from_pretrained("""allenai/led-base-16384"""  )




      @cached_property
      def 				lowercase__    (			self			: str  )				->      Optional[int]:
            return LEDTokenizerFast.from_pretrained("""allenai/led-base-16384"""  )




      @require_torch
      def 				lowercase__    (			self			: str  )				->      int:
            _lowerCAmelCase							     =							["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
            _lowerCAmelCase							     =							[0, 2_50, 2_51, 1_78_18, 13, 3_91_86, 19_38, 4, 2]

            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  _lowerCAmelCase							     =							tokenizer(__snake_case			,			max_length=len(__snake_case  )			,			padding=__snake_case			,			return_tensors="""pt"""  )
                  self.assertIsInstance(__snake_case			,			__snake_case  )

                  self.assertEqual((2, 9)			,			batch.input_ids.shape  )
                  self.assertEqual((2, 9)			,			batch.attention_mask.shape  )
                  _lowerCAmelCase							     =							batch.input_ids.tolist()[0]
                  self.assertListEqual(__snake_case			,			__snake_case  )




      @require_torch
      def 				lowercase__    (			self			: Tuple  )				->      Any:
            _lowerCAmelCase							     =							["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  _lowerCAmelCase							     =							tokenizer(__snake_case			,			padding=__snake_case			,			return_tensors="""pt"""  )
                  self.assertIn("""input_ids"""			,			__snake_case  )
                  self.assertIn("""attention_mask"""			,			__snake_case  )
                  self.assertNotIn("""labels"""			,			__snake_case  )
                  self.assertNotIn("""decoder_attention_mask"""			,			__snake_case  )




      @require_torch
      def 				lowercase__    (			self			: Optional[int]  )				->      List[Any]:
            _lowerCAmelCase							     =							[
                """Summary of the text.""",
                """Another summary.""",
            ]
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  _lowerCAmelCase							     =							tokenizer(text_target=__snake_case			,			max_length=32			,			padding="""max_length"""			,			return_tensors="""pt"""  )
                  self.assertEqual(32			,			targets["""input_ids"""].shape[1]  )




      @require_torch
      def 				lowercase__    (			self			: Tuple  )				->      Dict:
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  _lowerCAmelCase							     =							tokenizer(
                      ["""I am a small frog""" * 10_24, """I am a small frog"""]			,			padding=__snake_case			,			truncation=__snake_case			,			return_tensors="""pt"""  )
                  self.assertIsInstance(__snake_case			,			__snake_case  )
                  self.assertEqual(batch.input_ids.shape			,			(2, 51_22)  )




      @require_torch
      def 				lowercase__    (			self			: str  )				->      Any:
            _lowerCAmelCase							     =							["""A long paragraph for summarization."""]
            _lowerCAmelCase							     =							[
                """Summary of the text.""",
            ]
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  _lowerCAmelCase							     =							tokenizer(__snake_case			,			return_tensors="""pt"""  )
                  _lowerCAmelCase							     =							tokenizer(text_target=__snake_case			,			return_tensors="""pt"""  )
                  _lowerCAmelCase							     =							inputs["""input_ids"""]
                  _lowerCAmelCase							     =							targets["""input_ids"""]
                  self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()  )
                  self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()  )
                  self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()  )
                  self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()  )




      @require_torch
      def 				lowercase__    (			self			: str  )				->      Any:
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  _lowerCAmelCase							     =							["""Summary of the text.""", """Another summary."""]
                  _lowerCAmelCase							     =							[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

                  _lowerCAmelCase							     =							tokenizer(__snake_case			,			padding=__snake_case  )
                  _lowerCAmelCase							     =							[[0] * len(__snake_case  ) for x in encoded_output["""input_ids"""]]
                  _lowerCAmelCase							     =							tokenizer.pad(__snake_case  )
                  self.assertSequenceEqual(outputs["""global_attention_mask"""]			,			__snake_case  )




      def 				lowercase__    (			self			: Optional[int]  )				->      Union[str, Any]:
            pass




      def 				lowercase__    (			self			: int  )				->      Optional[Any]:
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"  ):
                        _lowerCAmelCase							     =							self.rust_tokenizer_class.from_pretrained(__snake_case			,			**__snake_case  )
                        _lowerCAmelCase							     =							self.tokenizer_class.from_pretrained(__snake_case			,			**__snake_case  )
                        _lowerCAmelCase							     =							"""A, <mask> AllenNLP sentence."""
                        _lowerCAmelCase							     =							tokenizer_r.encode_plus(__snake_case			,			add_special_tokens=__snake_case			,			return_token_type_ids=__snake_case  )
                        _lowerCAmelCase							     =							tokenizer_p.encode_plus(__snake_case			,			add_special_tokens=__snake_case			,			return_token_type_ids=__snake_case  )
                        self.assertEqual(sum(tokens_r["""token_type_ids"""]  )			,			sum(tokens_p["""token_type_ids"""]  )  )
                        self.assertEqual(
                            sum(tokens_r["""attention_mask"""]  ) / len(tokens_r["""attention_mask"""]  )			,			sum(tokens_p["""attention_mask"""]  ) / len(tokens_p["""attention_mask"""]  )			,			)

                        _lowerCAmelCase							     =							tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]  )
                        _lowerCAmelCase							     =							tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]  )
                        self.assertSequenceEqual(tokens_p["""input_ids"""]			,			[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]  )
                        self.assertSequenceEqual(tokens_r["""input_ids"""]			,			[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]  )

                        self.assertSequenceEqual(
                            __snake_case			,			["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]  )
                        self.assertSequenceEqual(
                            __snake_case			,			["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]  )













'''simple docstring'''


lowerCAmelCase					:     str												='''
# Installazione di Transformers
! pip install transformers datasets
# Per installare dalla fonte invece dell\'ultima versione rilasciata, commenta il comando sopra e
# rimuovi la modalità commento al comando seguente.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

lowerCAmelCase					:     int												=[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
lowerCAmelCase					:     List[str]												={
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}


