import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    EulerAncestralDiscreteScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionInstructPixaPixPipeline,
    UNetaDConditionModel,
)
from diffusers.image_processor import VaeImageProcessor
from diffusers.utils import floats_tensor, load_image, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()





class 		_lowercase						(					snake_case_						,      snake_case_						,      snake_case_						,      unittest.TestCase       ):
 lowercase				     =   StableDiffusionInstructPixaPixPipeline
 lowercase				     =   TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'height', 'width', 'cross_attention_kwargs'}
 lowercase				     =   TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
 lowercase				     =   IMAGE_TO_IMAGE_IMAGE_PARAMS
 lowercase				     =   IMAGE_TO_IMAGE_IMAGE_PARAMS

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Tuple			)   ->						List[str]:

  """simple docstring"""





  torch.manual_seed(0			)
  UpperCamelCase_					:						Dict =					UNetaDConditionModel(
      block_out_channels=(3_2, 6_4)   ,      layers_per_block=2   ,      sample_size=3_2   ,      in_channels=8   ,      out_channels=4   ,      down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D')   ,      up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D')   ,      cross_attention_dim=3_2   ,      )
  UpperCamelCase_					:						Optional[Any] =					PNDMScheduler(skip_prk_steps=snake_case			)
  torch.manual_seed(0			)
  UpperCamelCase_					:						Optional[int] =					AutoencoderKL(
      block_out_channels=[3_2, 6_4]   ,      in_channels=3   ,      out_channels=3   ,      down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']   ,      up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']   ,      latent_channels=4   ,      )
  torch.manual_seed(0			)
  UpperCamelCase_					:						Optional[int] =					CLIPTextConfig(
      bos_token_id=0   ,      eos_token_id=2   ,      hidden_size=3_2   ,      intermediate_size=3_7   ,      layer_norm_eps=1e-05   ,      num_attention_heads=4   ,      num_hidden_layers=5   ,      pad_token_id=1   ,      vocab_size=1_0_0_0   ,      )
  UpperCamelCase_					:						Union[str, Any] =					CLIPTextModel(snake_case			)
  UpperCamelCase_					:						Union[str, Any] =					CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'			)

  UpperCamelCase_					:						str =					{
      'unet': unet,
      'scheduler': scheduler,
      'vae': vae,
      'text_encoder': text_encoder,
      'tokenizer': tokenizer,
      'safety_checker': None,
      'feature_extractor': None,
  }
  return components

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Any   ,      snake_case	:							List[Any]   ,      snake_case	:							List[str]=0			)   ->						List[str]:

  """simple docstring"""





  UpperCamelCase_					:						List[str] =					floats_tensor((1, 3, 3_2, 3_2)   ,      rng=random.Random(snake_case			)			).to(snake_case			)
  UpperCamelCase_					:						List[str] =					image.cpu().permute(0   ,      2   ,      3   ,      1			)[0]
  UpperCamelCase_					:						Any =					Image.fromarray(np.uinta(snake_case			)			).convert('RGB'			)
  if str(snake_case			).startswith('mps'			):
   UpperCamelCase_					:						Any =					torch.manual_seed(snake_case			)
  else:
   UpperCamelCase_					:						Any =					torch.Generator(device=snake_case			).manual_seed(snake_case			)
  UpperCamelCase_					:						Any =					{
      'prompt': 'A painting of a squirrel eating a burger',
      'image': image,
      'generator': generator,
      'num_inference_steps': 2,
      'guidance_scale': 6.0,
      'image_guidance_scale': 1,
      'output_type': 'numpy',
  }
  return inputs

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Dict			)   ->						str:

  """simple docstring"""





  UpperCamelCase_					:						List[str] =					'cpu'  # ensure determinism for the device-dependent torch.Generator
  UpperCamelCase_					:						Optional[int] =					self.get_dummy_components()
  UpperCamelCase_					:						Union[str, Any] =					StableDiffusionInstructPixaPixPipeline(**snake_case			)
  UpperCamelCase_					:						List[str] =					sd_pipe.to(snake_case			)
  sd_pipe.set_progress_bar_config(disable=snake_case			)

  UpperCamelCase_					:						Union[str, Any] =					self.get_dummy_inputs(snake_case			)
  UpperCamelCase_					:						Any =					sd_pipe(**snake_case			).images
  UpperCamelCase_					:						List[str] =					image[0, -3:, -3:, -1]
  assert image.shape == (1, 3_2, 3_2, 3)
  UpperCamelCase_					:						str =					np.array([0.7526, 0.3750, 0.4547, 0.6117, 0.5866, 0.5016, 0.4327, 0.5642, 0.4815]			)

  assert np.abs(image_slice.flatten() - expected_slice			).max() < 1e-3

 def 						SCREAMING_SNAKE_CASE__   ( self	:							str			)   ->						int:

  """simple docstring"""





  UpperCamelCase_					:						int =					'cpu'  # ensure determinism for the device-dependent torch.Generator
  UpperCamelCase_					:						List[str] =					self.get_dummy_components()
  UpperCamelCase_					:						Dict =					StableDiffusionInstructPixaPixPipeline(**snake_case			)
  UpperCamelCase_					:						Any =					sd_pipe.to(snake_case			)
  sd_pipe.set_progress_bar_config(disable=snake_case			)

  UpperCamelCase_					:						Dict =					self.get_dummy_inputs(snake_case			)
  UpperCamelCase_					:						Dict =					'french fries'
  UpperCamelCase_					:						Optional[int] =					sd_pipe(**snake_case   ,      negative_prompt=snake_case			)
  UpperCamelCase_					:						List[str] =					output.images
  UpperCamelCase_					:						Dict =					image[0, -3:, -3:, -1]

  assert image.shape == (1, 3_2, 3_2, 3)
  UpperCamelCase_					:						Union[str, Any] =					np.array([0.7511, 0.3642, 0.4553, 0.6236, 0.5797, 0.5013, 0.4343, 0.5611, 0.4831]			)

  assert np.abs(image_slice.flatten() - expected_slice			).max() < 1e-3

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Optional[int]			)   ->						List[str]:

  """simple docstring"""





  UpperCamelCase_					:						Any =					'cpu'  # ensure determinism for the device-dependent torch.Generator
  UpperCamelCase_					:						Tuple =					self.get_dummy_components()
  UpperCamelCase_					:						Union[str, Any] =					StableDiffusionInstructPixaPixPipeline(**snake_case			)
  UpperCamelCase_					:						int =					sd_pipe.to(snake_case			)
  sd_pipe.set_progress_bar_config(disable=snake_case			)

  UpperCamelCase_					:						Tuple =					self.get_dummy_inputs(snake_case			)
  UpperCamelCase_					:						Optional[Any] =					[inputs['prompt']] * 2

  UpperCamelCase_					:						List[str] =					np.array(inputs['image']			).astype(np.floataa			) / 255.0
  UpperCamelCase_					:						Optional[Any] =					torch.from_numpy(snake_case			).unsqueeze(0			).to(snake_case			)
  UpperCamelCase_					:						Tuple =					image / 2 + 0.5
  UpperCamelCase_					:						Dict =					image.permute(0   ,      3   ,      1   ,      2			)
  UpperCamelCase_					:						Tuple =					image.repeat(2   ,      1   ,      1   ,      1			)

  UpperCamelCase_					:						int =					sd_pipe(**snake_case			).images
  UpperCamelCase_					:						List[Any] =					image[-1, -3:, -3:, -1]

  assert image.shape == (2, 3_2, 3_2, 3)
  UpperCamelCase_					:						Optional[Any] =					np.array([0.5812, 0.5748, 0.5222, 0.5908, 0.5695, 0.7174, 0.6804, 0.5523, 0.5579]			)

  assert np.abs(image_slice.flatten() - expected_slice			).max() < 1e-3

 def 						SCREAMING_SNAKE_CASE__   ( self	:							str			)   ->						int:

  """simple docstring"""





  UpperCamelCase_					:						Dict =					'cpu'  # ensure determinism for the device-dependent torch.Generator
  UpperCamelCase_					:						Union[str, Any] =					self.get_dummy_components()
  UpperCamelCase_					:						List[Any] =					EulerAncestralDiscreteScheduler(
      beta_start=0.00085   ,      beta_end=0.012   ,      beta_schedule='scaled_linear'			)
  UpperCamelCase_					:						int =					StableDiffusionInstructPixaPixPipeline(**snake_case			)
  UpperCamelCase_					:						Any =					sd_pipe.to(snake_case			)
  sd_pipe.set_progress_bar_config(disable=snake_case			)

  UpperCamelCase_					:						Dict =					self.get_dummy_inputs(snake_case			)
  UpperCamelCase_					:						Optional[Any] =					sd_pipe(**snake_case			).images
  UpperCamelCase_					:						Union[str, Any] =					image[0, -3:, -3:, -1]

  UpperCamelCase_					:						Dict =					[round(snake_case   ,      4			) for x in image_slice.flatten().tolist()]
  print(','.join([str(snake_case			) for x in slice]			)			)

  assert image.shape == (1, 3_2, 3_2, 3)
  UpperCamelCase_					:						Optional[Any] =					np.array([0.7417, 0.3842, 0.4732, 0.5776, 0.5891, 0.5139, 0.4052, 0.5673, 0.4986]			)

  assert np.abs(image_slice.flatten() - expected_slice			).max() < 1e-3

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Tuple			)   ->						List[Any]:

  """simple docstring"""





  super().test_inference_batch_single_identical(expected_max_diff=3e-3			)


 def 						SCREAMING_SNAKE_CASE__   ( self	:							Any			)   ->						Tuple:

  """simple docstring"""





  UpperCamelCase_					:						str =					self.get_dummy_components()
  UpperCamelCase_					:						Union[str, Any] =					StableDiffusionInstructPixaPixPipeline(**snake_case			)
  UpperCamelCase_					:						List[Any] =					VaeImageProcessor(do_resize=snake_case   ,      do_normalize=snake_case			)
  UpperCamelCase_					:						Any =					pipe.to(snake_case			)
  pipe.set_progress_bar_config(disable=snake_case			)

  UpperCamelCase_					:						Optional[Any] =					pipe(**self.get_dummy_inputs_by_type(snake_case   ,      input_image_type='pt'			)			)[0]

  UpperCamelCase_					:						List[Any] =					components['vae']
  UpperCamelCase_					:						Union[str, Any] =					self.get_dummy_inputs_by_type(snake_case   ,      input_image_type='pt'			)

  for image_param in self.image_latents_params:
   if image_param in inputs.keys():
    UpperCamelCase_					:						Optional[int] =					vae.encode(inputs[image_param]			).latent_dist.mode()

  UpperCamelCase_					:						Optional[Any] =					pipe(**snake_case			)[0]

  UpperCamelCase_					:						Any =					np.abs(out - out_latents_inputs			).max()
  self.assertLess(snake_case   ,      1e-4   ,      'passing latents as image input generate different result from passing image'			)





@slow
@require_torch_gpu
class 		_lowercase						(					unittest.TestCase       ):

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Union[str, Any]			)   ->						str:

  """simple docstring"""





  super().tearDown()
  gc.collect()
  torch.cuda.empty_cache()

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Optional[int]   ,      snake_case	:							Tuple=0			)   ->						List[Any]:

  """simple docstring"""





  UpperCamelCase_					:						Union[str, Any] =					torch.manual_seed(snake_case			)
  UpperCamelCase_					:						List[Any] =					load_image(
      'https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_pix2pix/example.jpg'			)
  UpperCamelCase_					:						Tuple =					{
      'prompt': 'turn him into a cyborg',
      'image': image,
      'generator': generator,
      'num_inference_steps': 3,
      'guidance_scale': 7.5,
      'image_guidance_scale': 1.0,
      'output_type': 'numpy',
  }
  return inputs

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Dict			)   ->						int:

  """simple docstring"""





  UpperCamelCase_					:						List[str] =					StableDiffusionInstructPixaPixPipeline.from_pretrained(
      'timbrooks/instruct-pix2pix'   ,      safety_checker=snake_case			)
  pipe.to(snake_case			)
  pipe.set_progress_bar_config(disable=snake_case			)
  pipe.enable_attention_slicing()

  UpperCamelCase_					:						str =					self.get_inputs()
  UpperCamelCase_					:						Union[str, Any] =					pipe(**snake_case			).images
  UpperCamelCase_					:						Dict =					image[0, -3:, -3:, -1].flatten()

  assert image.shape == (1, 5_1_2, 5_1_2, 3)
  UpperCamelCase_					:						Any =					np.array([0.5902, 0.6015, 0.6027, 0.5983, 0.6092, 0.6061, 0.5765, 0.5785, 0.5555]			)

  assert np.abs(expected_slice - image_slice			).max() < 1e-3

 def 						SCREAMING_SNAKE_CASE__   ( self	:							List[Any]			)   ->						Tuple:

  """simple docstring"""





  UpperCamelCase_					:						Any =					StableDiffusionInstructPixaPixPipeline.from_pretrained(
      'timbrooks/instruct-pix2pix'   ,      safety_checker=snake_case			)
  UpperCamelCase_					:						str =					LMSDiscreteScheduler.from_config(pipe.scheduler.config			)
  pipe.to(snake_case			)
  pipe.set_progress_bar_config(disable=snake_case			)
  pipe.enable_attention_slicing()

  UpperCamelCase_					:						Any =					self.get_inputs()
  UpperCamelCase_					:						List[str] =					pipe(**snake_case			).images
  UpperCamelCase_					:						Optional[Any] =					image[0, -3:, -3:, -1].flatten()

  assert image.shape == (1, 5_1_2, 5_1_2, 3)
  UpperCamelCase_					:						Optional[int] =					np.array([0.6578, 0.6817, 0.6972, 0.6761, 0.6856, 0.6916, 0.6428, 0.6516, 0.6301]			)

  assert np.abs(expected_slice - image_slice			).max() < 1e-3

 def 						SCREAMING_SNAKE_CASE__   ( self	:							int			)   ->						Any:

  """simple docstring"""





  UpperCamelCase_					:						int =					StableDiffusionInstructPixaPixPipeline.from_pretrained(
      'timbrooks/instruct-pix2pix'   ,      safety_checker=snake_case			)
  UpperCamelCase_					:						List[str] =					DDIMScheduler.from_config(pipe.scheduler.config			)
  pipe.to(snake_case			)
  pipe.set_progress_bar_config(disable=snake_case			)
  pipe.enable_attention_slicing()

  UpperCamelCase_					:						Optional[Any] =					self.get_inputs()
  UpperCamelCase_					:						List[str] =					pipe(**snake_case			).images
  UpperCamelCase_					:						List[Any] =					image[0, -3:, -3:, -1].flatten()

  assert image.shape == (1, 5_1_2, 5_1_2, 3)
  UpperCamelCase_					:						str =					np.array([0.3828, 0.3834, 0.3818, 0.3792, 0.3865, 0.3752, 0.3792, 0.3847, 0.3753]			)

  assert np.abs(expected_slice - image_slice			).max() < 1e-3

 def 						SCREAMING_SNAKE_CASE__   ( self	:							str			)   ->						List[Any]:

  """simple docstring"""





  UpperCamelCase_					:						int =					0

  def callback_fn(snake_case	:							int   ,      snake_case	:							int   ,      snake_case	:							torch.FloatTensor			) -> None:
   UpperCamelCase_					:						Tuple =					True
   nonlocal number_of_steps
   number_of_steps += 1
   if step == 1:
    UpperCamelCase_					:						Union[str, Any] =					latents.detach().cpu().numpy()
    assert latents.shape == (1, 4, 6_4, 6_4)
    UpperCamelCase_					:						Tuple =					latents[0, -3:, -3:, -1]
    UpperCamelCase_					:						List[str] =					np.array([-0.2463, -0.4644, -0.9756, 1.5176, 1.4414, 0.7866, 0.9897, 0.8521, 0.7983]			)

    assert np.abs(latents_slice.flatten() - expected_slice			).max() < 5e-2
   elif step == 2:
    UpperCamelCase_					:						Any =					latents.detach().cpu().numpy()
    assert latents.shape == (1, 4, 6_4, 6_4)
    UpperCamelCase_					:						int =					latents[0, -3:, -3:, -1]
    UpperCamelCase_					:						Dict =					np.array([-0.2644, -0.4626, -0.9653, 1.5176, 1.4551, 0.7686, 0.9805, 0.8452, 0.8115]			)

    assert np.abs(latents_slice.flatten() - expected_slice			).max() < 5e-2

  UpperCamelCase_					:						Tuple =					False

  UpperCamelCase_					:						Tuple =					StableDiffusionInstructPixaPixPipeline.from_pretrained(
      'timbrooks/instruct-pix2pix'   ,      safety_checker=snake_case   ,      torch_dtype=torch.floataa			)
  UpperCamelCase_					:						int =					pipe.to(snake_case			)
  pipe.set_progress_bar_config(disable=snake_case			)
  pipe.enable_attention_slicing()

  UpperCamelCase_					:						Optional[int] =					self.get_inputs()
  pipe(**snake_case   ,      callback=snake_case   ,      callback_steps=1			)
  assert callback_fn.has_been_called
  assert number_of_steps == 3

 def 						SCREAMING_SNAKE_CASE__   ( self	:							Tuple			)   ->						str:

  """simple docstring"""





  torch.cuda.empty_cache()
  torch.cuda.reset_max_memory_allocated()
  torch.cuda.reset_peak_memory_stats()

  UpperCamelCase_					:						Any =					StableDiffusionInstructPixaPixPipeline.from_pretrained(
      'timbrooks/instruct-pix2pix'   ,      safety_checker=snake_case   ,      torch_dtype=torch.floataa			)
  UpperCamelCase_					:						List[Any] =					pipe.to(snake_case			)
  pipe.set_progress_bar_config(disable=snake_case			)
  pipe.enable_attention_slicing(1			)
  pipe.enable_sequential_cpu_offload()

  UpperCamelCase_					:						Optional[Any] =					self.get_inputs()
  UpperCamelCase_					:						List[Any] =					pipe(**snake_case			)

  UpperCamelCase_					:						List[str] =					torch.cuda.max_memory_allocated()
  # make sure that less than 2.2 GB is allocated
  assert mem_bytes < 2.2 * 1_0**9


 def 						SCREAMING_SNAKE_CASE__   ( self	:							Tuple			)   ->						int:

  """simple docstring"""





  UpperCamelCase_					:						str =					self.get_inputs()
  # resize to resolution that is divisible by 8 but not 16 or 32
  UpperCamelCase_					:						Dict =					inputs['image'].resize((5_0_4, 5_0_4)			)

  UpperCamelCase_					:						int =					'timbrooks/instruct-pix2pix'
  UpperCamelCase_					:						Tuple =					StableDiffusionInstructPixaPixPipeline.from_pretrained(
      snake_case   ,      safety_checker=snake_case   ,      )
  pipe.to(snake_case			)
  pipe.set_progress_bar_config(disable=snake_case			)
  pipe.enable_attention_slicing()

  UpperCamelCase_					:						List[Any] =					pipe(**snake_case			)
  UpperCamelCase_					:						List[str] =					output.images[0]

  UpperCamelCase_					:						str =					image[2_5_5:2_5_8, 3_8_3:3_8_6, -1]

  assert image.shape == (5_0_4, 5_0_4, 3)
  UpperCamelCase_					:						Any =					np.array([0.2726, 0.2529, 0.2664, 0.2655, 0.2641, 0.2642, 0.2591, 0.2649, 0.2590]			)

  assert np.abs(image_slice.flatten() - expected_slice			).max() < 5e-3




import argparse

import torch

from transformers import GPTaLMHeadModel, RobertaForMaskedLM


if __name__ == "__main__":
   a_    					=       argparse.ArgumentParser(
       description=(
           'Extraction some layers of the full RobertaForMaskedLM or GPT2LMHeadModel for Transfer Learned'
           ' Distillation'
       )
   )
   parser.add_argument('--model_type', default='roberta', choices=['roberta', 'gpt2'])
   parser.add_argument('--model_name', default='roberta-large', type=str)
   parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_roberta_048131723.pth', type=str)
   parser.add_argument('--vocab_transform', action='store_true')
   a_    					=       parser.parse_args()

   if args.model_type == "roberta":
      a_    					=       RobertaForMaskedLM.from_pretrained(args.model_name)
      a_    					=       'roberta'
   elif args.model_type == "gpt2":
      a_    					=       GPTaLMHeadModel.from_pretrained(args.model_name)
      a_    					=       'transformer'

   a_    					=       model.state_dict()
   a_    					=       {}

   # Embeddings #
   if args.model_type == "gpt2":
      for param_name in ["wte.weight", "wpe.weight"]:
         a_    					=       state_dict[F"""{prefix}.{param_name}"""]
   else:
      for w in ["word_embeddings", "position_embeddings", "token_type_embeddings"]:
         a_    					=       F"""{prefix}.embeddings.{w}.weight"""
         a_    					=       state_dict[param_name]
      for w in ["weight", "bias"]:
         a_    					=       F"""{prefix}.embeddings.LayerNorm.{w}"""
         a_    					=       state_dict[param_name]

    # Transformer Blocks #
   a_    					=       0
   for teacher_idx in [0, 2, 4, 7, 9, 11]:
      if args.model_type == "gpt2":
         for layer in ["ln_1", "attn.c_attn", "attn.c_proj", "ln_2", "mlp.c_fc", "mlp.c_proj"]:
            for w in ["weight", "bias"]:
               a_    					=       state_dict[
                   F"""{prefix}.h.{teacher_idx}.{layer}.{w}"""
               ]
         a_    					=       state_dict[F"""{prefix}.h.{teacher_idx}.attn.bias"""]
      else:
         for layer in [
             "attention.self.query",
             "attention.self.key",
             "attention.self.value",
             "attention.output.dense",
             "attention.output.LayerNorm",
             "intermediate.dense",
             "output.dense",
             "output.LayerNorm",
         ]:
            for w in ["weight", "bias"]:
               a_    					=       state_dict[
                   F"""{prefix}.encoder.layer.{teacher_idx}.{layer}.{w}"""
               ]
      std_idx += 1

   # Language Modeling Head ###s
   if args.model_type == "roberta":
      for layer in ["lm_head.decoder.weight", "lm_head.bias"]:
         a_    					=       state_dict[F"""{layer}"""]
      if args.vocab_transform:
         for w in ["weight", "bias"]:
            a_    					=       state_dict[F"""lm_head.dense.{w}"""]
            a_    					=       state_dict[F"""lm_head.layer_norm.{w}"""]
   elif args.model_type == "gpt2":
      for w in ["weight", "bias"]:
         a_    					=       state_dict[F"""{prefix}.ln_f.{w}"""]
      a_    					=       state_dict['lm_head.weight']

   print(F"""N layers selected for distillation: {std_idx}""")
   print(F"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

   print(F"""Save transferred checkpoint to {args.dump_checkpoint}.""")
   torch.save(compressed_sd, args.dump_checkpoint)



