



from manim import *





class 		UpperCAmelCase    (    snake_case_					):





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	Rectangle(height=0.5							,  width=0.5      )
         _lowerCAmelCase			      =	Rectangle(height=0.46							,  width=0.46      ).set_stroke(width=0      )
         _lowerCAmelCase			      =	Rectangle(height=0.25							,  width=0.25      )

         _lowerCAmelCase			      =	[mem.copy() for i in range(6      )]
         _lowerCAmelCase			      =	[mem.copy() for i in range(6      )]
         _lowerCAmelCase			      =	VGroup(*_lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0      )
         _lowerCAmelCase			      =	VGroup(*_lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0      )
         _lowerCAmelCase			      =	VGroup(_lowerCAmelCase							,  _lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0      )
         _lowerCAmelCase			      =	Text('''CPU'''							,  font_size=24      )
         _lowerCAmelCase			      =	Group(_lowerCAmelCase							,  _lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0.5							,  aligned_edge=_lowerCAmelCase      )
         cpu.move_to([-2.5, -0.5, 0]      )
         self.add(_lowerCAmelCase      )

         _lowerCAmelCase			      =	[mem.copy() for i in range(4      )]
         _lowerCAmelCase			      =	VGroup(*_lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0      )
         _lowerCAmelCase			      =	Text('''GPU'''							,  font_size=24      )
         _lowerCAmelCase			      =	Group(_lowerCAmelCase							,  _lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0.5							,  aligned_edge=_lowerCAmelCase      )
         gpu.move_to([-1, -1, 0]      )
         self.add(_lowerCAmelCase      )

         _lowerCAmelCase			      =	[mem.copy() for i in range(6      )]
         _lowerCAmelCase			      =	VGroup(*_lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0      )

         _lowerCAmelCase			      =	Text('''Model'''							,  font_size=24      )
         _lowerCAmelCase			      =	Group(_lowerCAmelCase							,  _lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0.5							,  aligned_edge=_lowerCAmelCase      )
         model.move_to([3, -1.0, 0]      )
         self.add(_lowerCAmelCase      )

         _lowerCAmelCase			      =	[]
         _lowerCAmelCase			      =	[]

         for i, rect in enumerate(_lowerCAmelCase      ):
               rect.set_stroke(_lowerCAmelCase      )

               _lowerCAmelCase			      =	Rectangle(height=0.46 / 4							,  width=0.46 / 3      ).set_stroke(width=0.0      ).set_fill(_lowerCAmelCase							,  opacity=0.7      )

               if i == 0:
                     cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT      )							,  buff=0.02							,  direction=_lowerCAmelCase      )
                     cpu_target.set_x(cpu_target.get_x() + 0.1      )
               elif i == 3:
                     cpu_target.next_to(model_cpu_arr[0]							,  direction=_lowerCAmelCase							,  buff=0.0      )
               else:
                     cpu_target.next_to(model_cpu_arr[i - 1]							,  direction=_lowerCAmelCase							,  buff=0.0      )
               self.add(_lowerCAmelCase      )
               model_cpu_arr.append(_lowerCAmelCase      )

         self.add(*_lowerCAmelCase							,  *_lowerCAmelCase      )

         _lowerCAmelCase			      =	[meta_mem.copy() for i in range(6      )]
         _lowerCAmelCase			      =	[meta_mem.copy() for i in range(6      )]
         _lowerCAmelCase			      =	VGroup(*_lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0      )
         _lowerCAmelCase			      =	VGroup(*_lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0      )
         _lowerCAmelCase			      =	VGroup(_lowerCAmelCase							,  _lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0      )
         _lowerCAmelCase			      =	Text('''Disk'''							,  font_size=24      )
         _lowerCAmelCase			      =	Group(_lowerCAmelCase							,  _lowerCAmelCase      ).arrange(_lowerCAmelCase							,  buff=0.5							,  aligned_edge=_lowerCAmelCase      )
         disk.move_to([-4.0, -1.25, 0]      )
         self.add(_lowerCAmelCase							,  _lowerCAmelCase      )

         _lowerCAmelCase			      =	[]

         for i in range(6      ):
               _lowerCAmelCase			      =	fill.copy().set_fill(_lowerCAmelCase							,  opacity=0.8      )
               target.move_to(disk_left_col_base[i]      ).scale(0.5      )
               cpu_disk_arr.append(_lowerCAmelCase      )

         self.add(*_lowerCAmelCase      )

         _lowerCAmelCase			      =	Square(side_length=2.2      )
         key.move_to([-5, 2, 0]      )

         _lowerCAmelCase			      =	MarkupText(
             F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''							,  font_size=18							,  )

         key_text.move_to([-5, 2.4, 0]      )

         self.add(_lowerCAmelCase							,  _lowerCAmelCase      )

         _lowerCAmelCase			      =	MarkupText(
             F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''							,  font_size=18							,  )

         blue_text.next_to(_lowerCAmelCase							,  DOWN * 2.4							,  aligned_edge=key_text.get_left()      )
         self.add(_lowerCAmelCase      )

         _lowerCAmelCase			      =	MarkupText(F'''The offloaded weights are all sent to the CPU.'''							,  font_size=24      )
         step_a.move_to([2, 2, 0]      )

         self.play(Write(_lowerCAmelCase							,  run_time=3      )      )

         for i in range(6      ):
               _lowerCAmelCase			      =	cpu_disk_arr[i]
               _lowerCAmelCase			      =	rect.copy().set_fill(_lowerCAmelCase							,  opacity=0.8      ).scale(2.0      )
               cpa.generate_target()
               cpa.target.move_to(model_base[i]      )

               if i == 0:
                     rect.set_fill(_lowerCAmelCase							,  opacity=0.8      )
                     rect.generate_target()
                     rect.target.move_to(cpu_left_col_base[0]      ).scale(2.0      )

                     self.remove(
                         *_lowerCAmelCase							,  *_lowerCAmelCase							,  )

               else:
                     rect.generate_target()
                     rect.target.move_to(cpu_left_col_base[i]      ).scale(2.0      )
               self.play(MoveToTarget(_lowerCAmelCase      )							,  MoveToTarget(_lowerCAmelCase      )							,  model_base[i].animate.set_stroke(_lowerCAmelCase      )      )
         self.play(FadeOut(_lowerCAmelCase      )      )

         _lowerCAmelCase			      =	MarkupText(
             F'''Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed.'''							,  font_size=24							,  )
         step_a.move_to([2, 2, 0]      )

         self.play(Write(_lowerCAmelCase							,  run_time=3      )      )

         _lowerCAmelCase			      =	[]
         _lowerCAmelCase			      =	[]
         for i in range(6      ):
               _lowerCAmelCase			      =	Arrow(start=_lowerCAmelCase							,  end=_lowerCAmelCase							,  color=_lowerCAmelCase							,  buff=0.5      )
               a.next_to(model_base[i].get_left()							,  _lowerCAmelCase							,  buff=0.2      )
               arrows.append(_lowerCAmelCase      )
               animations.append(Write(_lowerCAmelCase      )      )
         self.play(*_lowerCAmelCase      )
         self.wait()




from typing import List, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_          =				logging.get_logger(__name__)

UpperCAmelCase_          =				{
    "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/config.json",
    # See all LED models at https://huggingface.co/models?filter=led
}





class 		UpperCAmelCase    (    snake_case_					):
   SCREAMING_SNAKE_CASE__					      =  '''led'''
   SCREAMING_SNAKE_CASE__					      =  {
       '''num_attention_heads''': '''encoder_attention_heads''',
       '''hidden_size''': '''d_model''',
       '''attention_probs_dropout_prob''': '''attention_dropout''',
       '''initializer_range''': '''init_std''',
   }





   def __init__(						self							,  _lowerCAmelCase=50_265							,  _lowerCAmelCase=16_384							,  _lowerCAmelCase=1_024							,  _lowerCAmelCase=12							,  _lowerCAmelCase=4_096							,  _lowerCAmelCase=16							,  _lowerCAmelCase=12							,  _lowerCAmelCase=4_096							,  _lowerCAmelCase=16							,  _lowerCAmelCase=0.0							,  _lowerCAmelCase=0.0							,  _lowerCAmelCase=True							,  _lowerCAmelCase=True							,  _lowerCAmelCase="gelu"							,  _lowerCAmelCase=1_024							,  _lowerCAmelCase=0.1							,  _lowerCAmelCase=0.0							,  _lowerCAmelCase=0.0							,  _lowerCAmelCase=0.02							,  _lowerCAmelCase=2							,  _lowerCAmelCase=0.0							,  _lowerCAmelCase=1							,  _lowerCAmelCase=0							,  _lowerCAmelCase=2							,  _lowerCAmelCase = 512							,  **_lowerCAmelCase							,  ):
         _lowerCAmelCase			      =	vocab_size
         _lowerCAmelCase			      =	max_encoder_position_embeddings
         _lowerCAmelCase			      =	max_decoder_position_embeddings
         _lowerCAmelCase			      =	d_model
         _lowerCAmelCase			      =	encoder_ffn_dim
         _lowerCAmelCase			      =	encoder_layers
         _lowerCAmelCase			      =	encoder_attention_heads
         _lowerCAmelCase			      =	decoder_ffn_dim
         _lowerCAmelCase			      =	decoder_layers
         _lowerCAmelCase			      =	decoder_attention_heads
         _lowerCAmelCase			      =	dropout
         _lowerCAmelCase			      =	attention_dropout
         _lowerCAmelCase			      =	activation_dropout
         _lowerCAmelCase			      =	activation_function
         _lowerCAmelCase			      =	init_std
         _lowerCAmelCase			      =	encoder_layerdrop
         _lowerCAmelCase			      =	decoder_layerdrop
         _lowerCAmelCase			      =	classifier_dropout
         _lowerCAmelCase			      =	use_cache
         _lowerCAmelCase			      =	encoder_layers
         _lowerCAmelCase			      =	attention_window

         super().__init__(
             pad_token_id=_lowerCAmelCase							,  bos_token_id=_lowerCAmelCase							,  eos_token_id=_lowerCAmelCase							,  is_encoder_decoder=_lowerCAmelCase							,  decoder_start_token_id=_lowerCAmelCase							,  **_lowerCAmelCase							,  )