code: 
  
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  import argparse
  import gc
  import os
  import threading
  
  import evaluate
  import psutil
  import torch
  from datasets import load_dataset
  from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
  from torch.utils.data import DataLoader
  from transformers import (
      AutoModelForSequenceClassification,
      AutoTokenizer,
      get_linear_schedule_with_warmup,
      set_seed,
  )
  
  from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin
  
  
  ########################################################################
  # This is a fully working simple example to use Accelerate
  #
  # This example trains a Bert base model on GLUE MRPC
  # in any of the following settings (with the same script):
  #   - single CPU or single GPU
  #   - multi GPUS (using PyTorch distributed mode)
  #   - (multi) TPUs
  #   - fp16 (mixed-precision) or fp32 (normal precision)
  #   - FSDP
  #
  # This example also demonstrates the checkpointing and sharding capabilities
  #
  # To run it in each of these various modes, follow the instructions
  # in the readme for examples:
  # https://github.com/huggingface/accelerate/tree/main/examples
  #
  ########################################################################
  
  
  __snake_case							      =						16
  __snake_case							      =						32
  
  
  def 						_A     (       SCREAMING_SNAKE_CASE__						:			Optional[Any] ):
      return int(x / 2**20 )
  
  
  
  
  
  
  
  class     UpperCAmelCase_		:
  
  
  
        """simple docstring"""
        def __enter__(       self				)       ->   Optional[Any]:
            gc.collect()
            torch.cuda.empty_cache()
            torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
            UpperCamelCase :Union[str, Any]										=     torch.cuda.memory_allocated()
            UpperCamelCase :int										=     psutil.Process()
  
            UpperCamelCase :Union[str, Any]										=     self.cpu_mem_used()
            UpperCamelCase :Tuple										=     True
            UpperCamelCase :Optional[int]										=     threading.Thread(target=self.peak_monitor_func				)
            UpperCamelCase :Union[str, Any]										=     True
            peak_monitor_thread.start()
            return self
        def 					UpperCAmelCase		(       self				)       ->   Optional[Any]:
            return self.process.memory_info().rss
        def 					UpperCAmelCase		(       self				)       ->   Any:
            UpperCamelCase :Dict										=     -1
  
            while True:
                UpperCamelCase :Union[str, Any]										=     max(self.cpu_mem_used()					,       self.cpu_peak				)
  
                # can't sleep or will not catch the peak right (this comment is here on purpose)
                # time.sleep(0.001) # 1msec
  
                if not self.peak_monitoring:
                    break
  
  
  
  
        def __exit__(       self					,       *SCREAMING_SNAKE_CASE_				)       ->   List[str]:
            UpperCamelCase :List[str]										=     False
  
            gc.collect()
            torch.cuda.empty_cache()
            UpperCamelCase :List[Any]										=     torch.cuda.memory_allocated()
            UpperCamelCase :Dict										=     torch.cuda.max_memory_allocated()
            UpperCamelCase :Union[str, Any]										=     bamb(self.end - self.begin				)
            UpperCamelCase :Any										=     bamb(self.peak - self.begin				)
  
            UpperCamelCase :List[Any]										=     self.cpu_mem_used()
            UpperCamelCase :str										=     bamb(self.cpu_end - self.cpu_begin				)
            UpperCamelCase :Optional[int]										=     bamb(self.cpu_peak - self.cpu_begin				)
            # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")
  
  
  # For testing only
  if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
   from accelerate.test_utils.training import mocked_dataloaders
  
   __snake_case							      =						mocked_dataloaders  # noqa: F811
  
  
  def 						_A     (       SCREAMING_SNAKE_CASE__						:			List[Any]  ,						SCREAMING_SNAKE_CASE__						:			Dict ):
      # For testing only
      if os.environ.get('''TESTING_MOCKED_DATALOADERS'''  ,						SCREAMING_SNAKE_CASE__ ) == "1":
          UpperCamelCase :Any										=     2
  
      # New Code #
      # Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
      UpperCamelCase :Dict										=     FullyShardedDataParallelPlugin(
          state_dict_config=FullStateDictConfig(offload_to_cpu=SCREAMING_SNAKE_CASE__  ,						ranka_only=SCREAMING_SNAKE_CASE__ )  ,						optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=SCREAMING_SNAKE_CASE__  ,						ranka_only=SCREAMING_SNAKE_CASE__ )  ,						)
  
      # Initialize accelerator
      if args.with_tracking:
          UpperCamelCase :Dict										=     Accelerator(
              cpu=args.cpu  ,						mixed_precision=args.mixed_precision  ,						log_with='''wandb'''  ,						project_dir=args.logging_dir  ,						fsdp_plugin=SCREAMING_SNAKE_CASE__  ,						)
      else:
          UpperCamelCase :int										=     Accelerator(fsdp_plugin=SCREAMING_SNAKE_CASE__ )
      accelerator.print(accelerator.distributed_type )
  
      if hasattr(args.checkpointing_steps  ,						'''isdigit''' ):
          if args.checkpointing_steps == "epoch":
              UpperCamelCase :Tuple										=     args.checkpointing_steps
          elif args.checkpointing_steps.isdigit():
              UpperCamelCase :Optional[Any]										=     int(args.checkpointing_steps )
          else:
              raise ValueError(
                  F'''Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.''' )
      else:
          UpperCamelCase :int										=     None
      # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
      UpperCamelCase :Any										=     config['''lr''']
      UpperCamelCase :Tuple										=     int(config['''num_epochs'''] )
      UpperCamelCase :Union[str, Any]										=     int(config['''seed'''] )
      UpperCamelCase :Union[str, Any]										=     int(config['''batch_size'''] )
  
      # We need to initialize the trackers we use, and also store our configuration
      if args.with_tracking:
          UpperCamelCase :Tuple										=     vars(SCREAMING_SNAKE_CASE__ )
          accelerator.init_trackers('''fsdp_glue_no_trainer'''  ,						SCREAMING_SNAKE_CASE__ )
  
      UpperCamelCase :Optional[Any]										=     AutoTokenizer.from_pretrained(args.model_name_or_path )
      UpperCamelCase :List[str]										=     load_dataset('''glue'''  ,						'''mrpc''' )
      UpperCamelCase :List[str]										=     evaluate.load('''glue'''  ,						'''mrpc''' )
  
      def tokenize_function(SCREAMING_SNAKE_CASE__						:			List[Any] ):
          # max_length=None => use the model max length (it's actually the default)
          UpperCamelCase :int										=     tokenizer(examples['''sentence1''']  ,						examples['''sentence2''']  ,						truncation=SCREAMING_SNAKE_CASE__  ,						max_length=SCREAMING_SNAKE_CASE__ )
          return outputs
  
      # Apply the method we just defined to all the examples in all the splits of the dataset
      # starting with the main process first:
      with accelerator.main_process_first():
          UpperCamelCase :int										=     datasets.map(
              SCREAMING_SNAKE_CASE__  ,						batched=SCREAMING_SNAKE_CASE__  ,						remove_columns=['''idx''', '''sentence1''', '''sentence2''']  ,						)
  
      # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
      # transformers library
      UpperCamelCase :str										=     tokenized_datasets.rename_column('''label'''  ,						'''labels''' )
  
      # If the batch size is too big we use gradient accumulation
      UpperCamelCase :Union[str, Any]										=     1
      if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
          UpperCamelCase :Union[str, Any]										=     batch_size // MAX_GPU_BATCH_SIZE
          UpperCamelCase :Dict										=     MAX_GPU_BATCH_SIZE
  
      def collate_fn(SCREAMING_SNAKE_CASE__						:			List[Any] ):
          # On TPU it's best to pad everything to the same length or training will be very slow.
          UpperCamelCase :Union[str, Any]										=     128 if accelerator.distributed_type == DistributedType.TPU else None
          # When using mixed precision we want round multiples of 8/16
          if accelerator.mixed_precision == "fp8":
              UpperCamelCase :List[Any]										=     16
          elif accelerator.mixed_precision != "no":
              UpperCamelCase :int										=     8
          else:
              UpperCamelCase :Dict										=     None
  
          return tokenizer.pad(
              SCREAMING_SNAKE_CASE__  ,						padding='''longest'''  ,						max_length=SCREAMING_SNAKE_CASE__  ,						pad_to_multiple_of=SCREAMING_SNAKE_CASE__  ,						return_tensors='''pt'''  ,						)
  
      # Instantiate dataloaders.
      UpperCamelCase :Optional[Any]										=     DataLoader(
          tokenized_datasets['''train''']  ,						shuffle=SCREAMING_SNAKE_CASE__  ,						collate_fn=SCREAMING_SNAKE_CASE__  ,						batch_size=SCREAMING_SNAKE_CASE__ )
      UpperCamelCase :int										=     DataLoader(
          tokenized_datasets['''validation''']  ,						shuffle=SCREAMING_SNAKE_CASE__  ,						collate_fn=SCREAMING_SNAKE_CASE__  ,						batch_size=SCREAMING_SNAKE_CASE__ )
  
      set_seed(SCREAMING_SNAKE_CASE__ )
  
      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      UpperCamelCase :List[Any]										=     AutoModelForSequenceClassification.from_pretrained(
          args.model_name_or_path  ,						return_dict=SCREAMING_SNAKE_CASE__  ,						low_cpu_mem_usage=SCREAMING_SNAKE_CASE__ )
  
      # New Code #
      # For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
      UpperCamelCase :int										=     accelerator.prepare(SCREAMING_SNAKE_CASE__ )
      accelerator.print(SCREAMING_SNAKE_CASE__ )
  
      # Instantiate optimizer
      # New Code #
      # For FSDP feature, at present it doesn't support multiple parameter groups,
      # so we need to create a single parameter group for the whole model
      UpperCamelCase :str										=     torch.optim.AdamW(params=model.parameters()  ,						lr=SCREAMING_SNAKE_CASE__  ,						weight_decay=2e-4 )
  
      # Instantiate scheduler
      UpperCamelCase :Optional[Any]										=     get_linear_schedule_with_warmup(
          optimizer=SCREAMING_SNAKE_CASE__  ,						num_warmup_steps=10  ,						num_training_steps=(len(SCREAMING_SNAKE_CASE__ ) * num_epochs) // gradient_accumulation_steps  ,						)
  
      # New Code #
      # For FSDP feature, prepare everything except the model as we have already prepared the model
      # before creating the optimizer
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      UpperCamelCase :List[Any]										=     accelerator.prepare(
          SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ )
  
      UpperCamelCase :List[Any]										=     0
  
      # Potentially load in the weights and states from a previous save
      if args.resume_from_checkpoint:
          if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
              accelerator.print(F'''Resumed from checkpoint: {args.resume_from_checkpoint}''' )
              accelerator.load_state(args.resume_from_checkpoint )
              UpperCamelCase :List[str]										=     os.path.basename(args.resume_from_checkpoint )
          else:
              # Get the most recent checkpoint
              UpperCamelCase :Optional[int]										=     [f.name for f in os.scandir(os.getcwd() ) if f.is_dir()]
              dirs.sort(key=os.path.getctime )
              UpperCamelCase :Dict										=     dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
          # Extract `epoch_{i}` or `step_{i}`
          UpperCamelCase :Dict										=     os.path.splitext(SCREAMING_SNAKE_CASE__ )[0]
  
          if "epoch" in training_difference:
              num_epochs -= int(training_difference.replace('''epoch_'''  ,						'''''' ) )
              UpperCamelCase :Dict										=     None
          else:
              UpperCamelCase :str										=     int(training_difference.replace('''step_'''  ,						'''''' ) )
              num_epochs -= resume_step // len(SCREAMING_SNAKE_CASE__ )
              # If resuming by step, we also need to know exactly how far into the DataLoader we went
              UpperCamelCase :List[str]										=     (num_epochs * len(SCREAMING_SNAKE_CASE__ )) - resume_step
  
      # Now we train the model
      for epoch in range(SCREAMING_SNAKE_CASE__ ):
          # New Code #
          # context manager to track the peak memory usage during the training epoch
          with TorchTracemalloc() as tracemalloc:
              model.train()
              if args.with_tracking:
                  UpperCamelCase :List[str]										=     0
              for step, batch in enumerate(SCREAMING_SNAKE_CASE__ ):
                  # We need to skip steps until we reach the resumed step
                  if args.resume_from_checkpoint and epoch == 0:
                      if resume_step is not None and step < resume_step:
                          pass
                  # We could avoid this line since we set the accelerator with `device_placement=True`.
                  batch.to(accelerator.device )
                  UpperCamelCase :Optional[Any]										=     model(**SCREAMING_SNAKE_CASE__ )
                  UpperCamelCase :Dict										=     outputs.loss
                  UpperCamelCase :Tuple										=     loss / gradient_accumulation_steps
                  # We keep track of the loss at each epoch
                  if args.with_tracking:
                      total_loss += loss.detach().float()
                  accelerator.backward(SCREAMING_SNAKE_CASE__ )
                  if step % gradient_accumulation_steps == 0:
                      optimizer.step()
                      lr_scheduler.step()
                      optimizer.zero_grad()
                      # accelerator.print(lr_scheduler.get_lr())
  
                  overall_step += 1
  
                  if isinstance(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ ):
                      UpperCamelCase :Any										=     F'''step_{overall_step}'''
                      if overall_step % checkpointing_steps == 0:
                          if args.output_dir is not None:
                              UpperCamelCase :Optional[Any]										=     os.path.join(args.output_dir  ,						SCREAMING_SNAKE_CASE__ )
                          accelerator.save_state(SCREAMING_SNAKE_CASE__ )
          # New Code #
          # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
          accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin ) ) )
          accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used ) )
          accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked ) )
          accelerator.print(
              '''Total Peak Memory consumed during the train (max): {}'''.format(
                  tracemalloc.peaked + bamb(tracemalloc.begin ) ) )
          # Logging the peak memory usage of the GPU to the tracker
          if args.with_tracking:
              accelerator.log(
                  {
                      '''train_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin ),
                  }  ,						step=SCREAMING_SNAKE_CASE__  ,						)
  
          # New Code #
          # context manager to track the peak memory usage during the evaluation
          with TorchTracemalloc() as tracemalloc:
              model.eval()
              for step, batch in enumerate(SCREAMING_SNAKE_CASE__ ):
                  # We could avoid this line since we set the accelerator with `device_placement=True`.
                  batch.to(accelerator.device )
                  with torch.no_grad():
                      UpperCamelCase :Union[str, Any]										=     model(**SCREAMING_SNAKE_CASE__ )
                  UpperCamelCase :List[str]										=     outputs.logits.argmax(dim=-1 )
                  UpperCamelCase :List[str]										=     accelerator.gather_for_metrics((predictions, batch['''labels''']) )
                  metric.add_batch(
                      predictions=SCREAMING_SNAKE_CASE__  ,						references=SCREAMING_SNAKE_CASE__  ,						)
  
              UpperCamelCase :Union[str, Any]										=     metric.compute()
              # Use accelerator.print to print only on the main process.
              accelerator.print(F'''epoch {epoch}:'''  ,						SCREAMING_SNAKE_CASE__ )
              if args.with_tracking:
                  accelerator.log(
                      {
                          '''accuracy''': eval_metric['''accuracy'''],
                          '''f1''': eval_metric['''f1'''],
                          '''train_loss''': total_loss.item() / len(SCREAMING_SNAKE_CASE__ ),
                      }  ,						step=SCREAMING_SNAKE_CASE__  ,						)
  
              if checkpointing_steps == "epoch":
                  UpperCamelCase :Optional[int]										=     F'''epoch_{epoch}'''
                  if args.output_dir is not None:
                      UpperCamelCase :Any										=     os.path.join(args.output_dir  ,						SCREAMING_SNAKE_CASE__ )
                  accelerator.save_state(SCREAMING_SNAKE_CASE__ )
          # New Code #
          # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
          accelerator.print('''Memory before entering the eval : {}'''.format(bamb(tracemalloc.begin ) ) )
          accelerator.print('''Memory consumed at the end of the eval (end-begin): {}'''.format(tracemalloc.used ) )
          accelerator.print('''Peak Memory consumed during the eval (max-begin): {}'''.format(tracemalloc.peaked ) )
          accelerator.print(
              '''Total Peak Memory consumed during the eval (max): {}'''.format(tracemalloc.peaked + bamb(tracemalloc.begin ) ) )
          # Logging the peak memory usage of the GPU to the tracker
          if args.with_tracking:
              accelerator.log(
                  {
                      '''eval_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin ),
                  }  ,						step=SCREAMING_SNAKE_CASE__  ,						)
  
      if args.with_tracking:
          accelerator.end_training()
  
  
  def 						_A     (       ):
      UpperCamelCase :str										=     argparse.ArgumentParser(description='''Simple example of training script.''' )
      parser.add_argument(
          '''--mixed_precision'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default=SCREAMING_SNAKE_CASE__  ,						choices=['''no''', '''fp16''', '''bf16''', '''fp8''']  ,						help='''Whether to use mixed precision. Choose'''
          '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
          '''and an Nvidia Ampere GPU.'''  ,						)
      parser.add_argument('''--cpu'''  ,						action='''store_true'''  ,						help='''If passed, will train on the CPU.''' )
      parser.add_argument(
          '''--checkpointing_steps'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default=SCREAMING_SNAKE_CASE__  ,						help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''  ,						)
      parser.add_argument(
          '''--resume_from_checkpoint'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default=SCREAMING_SNAKE_CASE__  ,						help='''If the training should continue from a checkpoint folder.'''  ,						)
      parser.add_argument(
          '''--with_tracking'''  ,						action='''store_true'''  ,						help='''Whether to load in all available experiment trackers from the environment and use them for logging.'''  ,						)
      parser.add_argument(
          '''--output_dir'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default='''.'''  ,						help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''  ,						)
      parser.add_argument(
          '''--logging_dir'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default='''logs'''  ,						help='''Location on where to store experiment tracking logs`'''  ,						)
      parser.add_argument(
          '''--model_name_or_path'''  ,						type=SCREAMING_SNAKE_CASE__  ,						help='''Path to pretrained model or model identifier from huggingface.co/models.'''  ,						required=SCREAMING_SNAKE_CASE__  ,						)
      UpperCamelCase :str										=     parser.parse_args()
      UpperCamelCase :List[Any]										=     {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
      training_function(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ )
  
  
  if __name__ == "__main__":
   main()
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  from __future__ import annotations
  
  import inspect
  import unittest
  
  import numpy as np
  
  from transformers import SwinConfig
  from transformers.testing_utils import require_tf, require_vision, slow, to_atuple
  from transformers.utils import cached_property, is_tf_available, is_vision_available
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_tf_available():
   import tensorflow as tf
  
   from transformers.models.swin.modeling_tf_swin import (
       TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
       TFSwinForImageClassification,
       TFSwinForMaskedImageModeling,
       TFSwinModel,
   )
  
  
  if is_vision_available():
   from PIL import Image
  
   from transformers import AutoImageProcessor
  
  
  
  
  
  
  
  class     UpperCAmelCase_		:
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_=13					,       SCREAMING_SNAKE_CASE_=32					,       SCREAMING_SNAKE_CASE_=2					,       SCREAMING_SNAKE_CASE_=3					,       SCREAMING_SNAKE_CASE_=16					,       SCREAMING_SNAKE_CASE_=[1, 2, 1]					,       SCREAMING_SNAKE_CASE_=[2, 2, 4]					,       SCREAMING_SNAKE_CASE_=2					,       SCREAMING_SNAKE_CASE_=2.0					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=0.0					,       SCREAMING_SNAKE_CASE_=0.0					,       SCREAMING_SNAKE_CASE_=0.1					,       SCREAMING_SNAKE_CASE_="gelu"					,       SCREAMING_SNAKE_CASE_=False					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=0.02					,       SCREAMING_SNAKE_CASE_=1e-5					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=None					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=10					,       SCREAMING_SNAKE_CASE_=8					,       )       ->   None:
            UpperCamelCase :Optional[Any]										=     parent
            UpperCamelCase :Optional[Any]										=     batch_size
            UpperCamelCase :List[Any]										=     image_size
            UpperCamelCase :List[Any]										=     patch_size
            UpperCamelCase :List[Any]										=     num_channels
            UpperCamelCase :Optional[Any]										=     embed_dim
            UpperCamelCase :List[Any]										=     depths
            UpperCamelCase :List[Any]										=     num_heads
            UpperCamelCase :Tuple										=     window_size
            UpperCamelCase :Optional[int]										=     mlp_ratio
            UpperCamelCase :int										=     qkv_bias
            UpperCamelCase :str										=     hidden_dropout_prob
            UpperCamelCase :Dict										=     attention_probs_dropout_prob
            UpperCamelCase :Dict										=     drop_path_rate
            UpperCamelCase :Optional[Any]										=     hidden_act
            UpperCamelCase :int										=     use_absolute_embeddings
            UpperCamelCase :Any										=     patch_norm
            UpperCamelCase :str										=     layer_norm_eps
            UpperCamelCase :List[Any]										=     initializer_range
            UpperCamelCase :str										=     is_training
            UpperCamelCase :Union[str, Any]										=     scope
            UpperCamelCase :List[str]										=     use_labels
            UpperCamelCase :Union[str, Any]										=     type_sequence_label_size
            UpperCamelCase :int										=     encoder_stride
        def 					UpperCAmelCase		(       self				)       ->   Optional[int]:
            UpperCamelCase :Dict										=     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)
  
            UpperCamelCase :List[Any]										=     None
            if self.use_labels:
                UpperCamelCase :Tuple										=     ids_tensor([self.batch_size]					,       self.type_sequence_label_size				)
  
            UpperCamelCase :List[str]										=     self.get_config()
  
            return config, pixel_values, labels
        def 					UpperCAmelCase		(       self				)       ->   int:
            return SwinConfig(
                image_size=self.image_size					,       patch_size=self.patch_size					,       num_channels=self.num_channels					,       embed_dim=self.embed_dim					,       depths=self.depths					,       num_heads=self.num_heads					,       window_size=self.window_size					,       mlp_ratio=self.mlp_ratio					,       qkv_bias=self.qkv_bias					,       hidden_dropout_prob=self.hidden_dropout_prob					,       attention_probs_dropout_prob=self.attention_probs_dropout_prob					,       drop_path_rate=self.drop_path_rate					,       hidden_act=self.hidden_act					,       use_absolute_embeddings=self.use_absolute_embeddings					,       path_norm=self.patch_norm					,       layer_norm_eps=self.layer_norm_eps					,       initializer_range=self.initializer_range					,       encoder_stride=self.encoder_stride					,       )
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   List[Any]:
            UpperCamelCase :int										=     TFSwinModel(config=SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Tuple										=     model(SCREAMING_SNAKE_CASE_				)
  
            UpperCamelCase :str										=     ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths				) - 1))
            UpperCamelCase :Union[str, Any]										=     int(config.embed_dim * 2 ** (len(config.depths				) - 1)				)
  
            self.parent.assertEqual(result.last_hidden_state.shape					,       (self.batch_size, expected_seq_len, expected_dim)				)
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   Any:
            UpperCamelCase :int										=     TFSwinForMaskedImageModeling(config=SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :List[str]										=     model(SCREAMING_SNAKE_CASE_				)
            self.parent.assertEqual(
                result.logits.shape					,       (self.batch_size, self.num_channels, self.image_size, self.image_size)				)
  
            # test greyscale images
            UpperCamelCase :Optional[int]										=     1
            UpperCamelCase :int										=     TFSwinForMaskedImageModeling(SCREAMING_SNAKE_CASE_				)
  
            UpperCamelCase :str										=     floats_tensor([self.batch_size, 1, self.image_size, self.image_size]				)
            UpperCamelCase :str										=     model(SCREAMING_SNAKE_CASE_				)
            self.parent.assertEqual(result.logits.shape					,       (self.batch_size, 1, self.image_size, self.image_size)				)
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   List[str]:
            UpperCamelCase :Optional[Any]										=     self.type_sequence_label_size
            UpperCamelCase :Optional[int]										=     TFSwinForImageClassification(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Tuple										=     model(SCREAMING_SNAKE_CASE_					,       labels=SCREAMING_SNAKE_CASE_				)
            self.parent.assertEqual(result.logits.shape					,       (self.batch_size, self.type_sequence_label_size)				)
  
            # test greyscale images
            UpperCamelCase :Union[str, Any]										=     1
            UpperCamelCase :Tuple										=     TFSwinForImageClassification(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Any										=     floats_tensor([self.batch_size, 1, self.image_size, self.image_size]				)
            UpperCamelCase :Tuple										=     model(SCREAMING_SNAKE_CASE_				)
            self.parent.assertEqual(result.logits.shape					,       (self.batch_size, self.type_sequence_label_size)				)
  
  
  
  
        def 					UpperCAmelCase		(       self				)       ->   int:
            UpperCamelCase :List[str]										=     self.prepare_config_and_inputs()
            UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase :Optional[Any]										=     config_and_inputs
            UpperCamelCase :Any										=     {'''pixel_values''': pixel_values}
            return config, inputs_dict
  
  
  
  
  
  
  
  @require_tf
  class     UpperCAmelCase_		(      lowercase,			lowercase,			unittest.TestCase				):
  
  
  
        """simple docstring"""
        UpperCamelCase_       :      int     =(
            (
                TFSwinModel,
                TFSwinForImageClassification,
                TFSwinForMaskedImageModeling,
            )
            if is_tf_available()
            else ()
        )
        UpperCamelCase_       :      Any     =(
            {'feature-extraction': TFSwinModel, 'image-classification': TFSwinForImageClassification}
            if is_tf_available()
            else {}
        )
  
        UpperCamelCase_       :      Optional[int]     =False
        UpperCamelCase_       :      int     =False
        UpperCamelCase_       :      Optional[int]     =False
        UpperCamelCase_       :      Any     =False
        def 					UpperCAmelCase		(       self				)       ->   Tuple:
            UpperCamelCase :str										=     TFSwinModelTester(self				)
            UpperCamelCase :str										=     ConfigTester(self					,       config_class=SCREAMING_SNAKE_CASE_					,       embed_dim=37				)
        def 					UpperCAmelCase		(       self				)       ->   Dict:
            self.create_and_test_config_common_properties()
            self.config_tester.create_and_test_config_to_json_string()
            self.config_tester.create_and_test_config_to_json_file()
            self.config_tester.create_and_test_config_from_and_save_pretrained()
            self.config_tester.create_and_test_config_with_num_labels()
            self.config_tester.check_config_can_be_init_without_params()
            self.config_tester.check_config_arguments_init()
        def 					UpperCAmelCase		(       self				)       ->   Union[str, Any]:
            return
        def 					UpperCAmelCase		(       self				)       ->   Optional[Any]:
            UpperCamelCase :Any										=     self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_				)
        def 					UpperCAmelCase		(       self				)       ->   Optional[int]:
            UpperCamelCase :Optional[int]										=     self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_masked_image_modeling(*SCREAMING_SNAKE_CASE_				)
        def 					UpperCAmelCase		(       self				)       ->   Optional[Any]:
            UpperCamelCase :Any										=     self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_image_classification(*SCREAMING_SNAKE_CASE_				)
        @unittest.skip(reason='''Swin does not use inputs_embeds'''				)
        def 					UpperCAmelCase		(       self				)       ->   Union[str, Any]:
            pass
        def 					UpperCAmelCase		(       self				)       ->   Optional[int]:
            UpperCamelCase							,     UpperCamelCase :int										=     self.model_tester.prepare_config_and_inputs_for_common()
  
            for model_class in self.all_model_classes:
                UpperCamelCase :Optional[int]										=     model_class(SCREAMING_SNAKE_CASE_				)
                self.assertIsInstance(model.get_input_embeddings()					,       tf.keras.layers.Layer				)
                UpperCamelCase :Union[str, Any]										=     model.get_output_embeddings()
                self.assertTrue(x is None or isinstance(SCREAMING_SNAKE_CASE_					,       tf.keras.layers.Dense				)				)
        def 					UpperCAmelCase		(       self				)       ->   str:
            UpperCamelCase							,     UpperCamelCase :str										=     self.model_tester.prepare_config_and_inputs_for_common()
  
            for model_class in self.all_model_classes:
                UpperCamelCase :Union[str, Any]										=     model_class(SCREAMING_SNAKE_CASE_				)
                UpperCamelCase :Union[str, Any]										=     inspect.signature(model.call				)
                # signature.parameters is an OrderedDict => so arg_names order is deterministic
                UpperCamelCase :Dict										=     [*signature.parameters.keys()]
  
                UpperCamelCase :Tuple										=     ['''pixel_values''']
                self.assertListEqual(arg_names[:1]					,       SCREAMING_SNAKE_CASE_				)
        def 					UpperCAmelCase		(       self				)       ->   List[Any]:
            UpperCamelCase							,     UpperCamelCase :Optional[int]										=     self.model_tester.prepare_config_and_inputs_for_common()
            UpperCamelCase :str										=     True
  
            for model_class in self.all_model_classes:
                UpperCamelCase :List[Any]										=     True
                UpperCamelCase :Optional[Any]										=     False
                UpperCamelCase :List[str]										=     True
                UpperCamelCase :Optional[int]										=     model_class(SCREAMING_SNAKE_CASE_				)
                UpperCamelCase :Union[str, Any]										=     model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)				)
                UpperCamelCase :List[Any]										=     outputs.attentions
                UpperCamelCase :int										=     len(self.model_tester.depths				)
                self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)
  
                # check that output_attentions also work using config
                del inputs_dict["output_attentions"]
                UpperCamelCase :Optional[Any]										=     True
                UpperCamelCase :List[Any]										=     config.window_size**2
                UpperCamelCase :Dict										=     model_class(SCREAMING_SNAKE_CASE_				)
                UpperCamelCase :Dict										=     model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)				)
                UpperCamelCase :List[Any]										=     outputs.attentions
                self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)
  
                self.assertListEqual(
                    list(attentions[0].shape[-3:]				)					,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,       )
                UpperCamelCase :List[Any]										=     len(SCREAMING_SNAKE_CASE_				)
  
                # Check attention is always last and order is fine
                UpperCamelCase :int										=     True
                UpperCamelCase :List[str]										=     True
                UpperCamelCase :List[str]										=     model_class(SCREAMING_SNAKE_CASE_				)
                UpperCamelCase :Tuple										=     model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)				)
  
                if hasattr(self.model_tester					,       '''num_hidden_states_types'''				):
                    UpperCamelCase :Union[str, Any]										=     self.model_tester.num_hidden_states_types
                else:
                    # also another +1 for reshaped_hidden_states
                    UpperCamelCase :Optional[int]										=     2
                self.assertEqual(out_len + added_hidden_states					,       len(SCREAMING_SNAKE_CASE_				)				)
  
                UpperCamelCase :Optional[int]										=     outputs.attentions
  
                self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)
  
                self.assertListEqual(
                    list(self_attentions[0].shape[-3:]				)					,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,       )
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   Dict:
            UpperCamelCase :Optional[int]										=     model_class(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Any										=     model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)				)
            UpperCamelCase :Union[str, Any]										=     outputs.hidden_states
  
            UpperCamelCase :Any										=     getattr(
                self.model_tester					,       '''expected_num_hidden_layers'''					,       len(self.model_tester.depths				) + 1				)
            self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)
  
            # Swin has a different seq_length
            UpperCamelCase :str										=     to_atuple(config.patch_size				)
  
            UpperCamelCase :Optional[int]										=     (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
  
            self.assertListEqual(
                list(hidden_states[0].shape[-2:]				)					,       [num_patches, self.model_tester.embed_dim]					,       )
  
            UpperCamelCase :str										=     outputs.reshaped_hidden_states
            self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)
  
            UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase :Optional[int]										=     reshaped_hidden_states[0].shape
  
            UpperCamelCase :int										=     tf.reshape(reshaped_hidden_states[0]					,       (batch_size, num_channels, height * width)				)
            UpperCamelCase :Dict										=     tf.transpose(SCREAMING_SNAKE_CASE_					,       (0, 2, 1)				)
  
            self.assertListEqual(
                list(reshaped_hidden_states.shape[-2:]				)					,       [num_patches, self.model_tester.embed_dim]					,       )
        def 					UpperCAmelCase		(       self				)       ->   str:
            UpperCamelCase							,     UpperCamelCase :int										=     self.model_tester.prepare_config_and_inputs_for_common()
  
            UpperCamelCase :List[Any]										=     to_atuple(self.model_tester.image_size				)
  
            for model_class in self.all_model_classes:
                UpperCamelCase :List[Any]										=     True
                self.check_hidden_states_output(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
  
                # check that output_hidden_states also work using config
                del inputs_dict["output_hidden_states"]
                UpperCamelCase :List[str]										=     True
  
                self.check_hidden_states_output(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
        def 					UpperCAmelCase		(       self				)       ->   Optional[Any]:
            UpperCamelCase							,     UpperCamelCase :List[Any]										=     self.model_tester.prepare_config_and_inputs_for_common()
            UpperCamelCase :List[Any]										=     3
  
            UpperCamelCase :Any										=     to_atuple(self.model_tester.image_size				)
            UpperCamelCase :List[Any]										=     to_atuple(config.patch_size				)
  
            UpperCamelCase :Dict										=     image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
            UpperCamelCase :List[str]										=     image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])
  
            for model_class in self.all_model_classes:
                UpperCamelCase :List[Any]										=     True
                self.check_hidden_states_output(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       (padded_height, padded_width)				)
  
                # check that output_hidden_states also work using config
                del inputs_dict["output_hidden_states"]
                UpperCamelCase :str										=     True
                self.check_hidden_states_output(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       (padded_height, padded_width)				)
  
  
  
  
        @slow
        def 					UpperCAmelCase		(       self				)       ->   List[Any]:
            for model_name in TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                UpperCamelCase :Dict										=     TFSwinModel.from_pretrained(SCREAMING_SNAKE_CASE_				)
                self.assertIsNotNone(SCREAMING_SNAKE_CASE_				)
  
  
  
  
  
  
  
  @require_vision
  @require_tf
  class     UpperCAmelCase_		(      unittest.TestCase				):
  
  
  
        """simple docstring"""
        @cached_property
        def 					UpperCAmelCase		(       self				)       ->   Union[str, Any]:
            return (
                AutoImageProcessor.from_pretrained('''microsoft/swin-tiny-patch4-window7-224'''				)
                if is_vision_available()
                else None
            )
  
  
  
  
        @slow
        def 					UpperCAmelCase		(       self				)       ->   Optional[int]:
            UpperCamelCase :Tuple										=     TFSwinForImageClassification.from_pretrained('''microsoft/swin-tiny-patch4-window7-224'''				)
            UpperCamelCase :int										=     self.default_image_processor
  
            UpperCamelCase :Optional[int]										=     Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''				)
            UpperCamelCase :List[str]										=     image_processor(images=SCREAMING_SNAKE_CASE_					,       return_tensors='''tf'''				)
  
            # forward pass
            UpperCamelCase :Optional[Any]										=     model(SCREAMING_SNAKE_CASE_				)
  
            # verify the logits
            UpperCamelCase :Tuple										=     tf.TensorShape((1, 1000)				)
            self.assertEqual(outputs.logits.shape					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :List[Any]										=     tf.constant([-0.0948, -0.6454, -0.0921]				)
            self.assertTrue(np.allclose(outputs.logits[0, :3]					,       SCREAMING_SNAKE_CASE_					,       atol=1e-4				)				)
  
  
  
  
  
  