code: 
  
  
  
  
  
  import logging
  import os
  import time
  from argparse import Namespace
  from pathlib import Path
  
  import datasets
  import torch
  from accelerate import Accelerator, DistributedType
  from accelerate.utils import ProjectConfiguration
  from arguments import TrainingArguments
  from datasets import load_dataset
  from huggingface_hub import Repository
  from torch.optim import AdamW
  from torch.utils.data import IterableDataset
  from torch.utils.data.dataloader import DataLoader
  from torch.utils.data.datapipes.iter.combinatorics import ShufflerIterDataPipe
  
  import transformers
  from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, get_scheduler, set_seed
  
  
  
  
  
  
  class 	UpperCamelCase_     (     snake_case__       ):
  
  
  
  					def __init__(		self    :    List[Any]					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Union[str, Any]=False					,			lowerCamelCase    :    Tuple=10_24					,			lowerCamelCase    :    int=10_24					,			lowerCamelCase    :    Any=3.6					,			lowerCamelCase    :    Dict=False					,			):
  							lowerCamelCase_  :    str   	=			tokenizer
  							lowerCamelCase_  :    str   	=			tokenizer.bos_token_id
  							lowerCamelCase_  :    Dict   	=			dataset
  							lowerCamelCase_  :    Dict   	=			seq_length
  							lowerCamelCase_  :    Optional[int]   	=			0
  							lowerCamelCase_  :    Union[str, Any]   	=			infinite
  							lowerCamelCase_  :    Optional[int]   	=			0
  							lowerCamelCase_  :    str   	=			tokenized
  
  							if self.tokenized:
  									lowerCamelCase_  :    int   	=			seq_length * num_of_sequences
  									lowerCamelCase_  :    List[str]   	=			'input_ids'
  							else:
  									lowerCamelCase_  :    str   	=			seq_length * chars_per_token * num_of_sequences
  									lowerCamelCase_  :    int   	=			'content'
  
  
  
  					def __iter__(		self    :    Optional[Any]	):
  							lowerCamelCase_  :    Tuple   	=			iter(self.dataset	)
  							lowerCamelCase_  :    Any   	=			True
  							while more_examples:
  									lowerCamelCase_	,			lowerCamelCase_  :    str   	=			[], 0
  									while True:
  											if buffer_len >= self.max_buffer_size:
  													break
  											try:
  													buffer.append(next(lowerCamelCase	)[self.content_field]	)
  													buffer_len += len(buffer[-1]	)
  											except StopIteration:
  													if self.infinite:
  															lowerCamelCase_  :    Any   	=			iter(self.dataset	)
  															self.epoch += 1
  															logger.info(F"Dataset epoch: {self.epoch}"	)
  													else:
  															lowerCamelCase_  :    List[str]   	=			False
  															break
  									if self.tokenized:
  											lowerCamelCase_  :    Optional[Any]   	=			buffer
  									else:
  											lowerCamelCase_  :    Union[str, Any]   	=			self.tokenizer(lowerCamelCase					,			truncation=lowerCamelCase	)['input_ids']
  									lowerCamelCase_  :    Union[str, Any]   	=			[]
  									for tokenized_input in tokenized_inputs:
  											all_token_ids.extend(tokenized_input + [self.concat_token_id]	)
  									for i in range(0					,			len(lowerCamelCase	)					,			self.seq_length	):
  											lowerCamelCase_  :    Optional[int]   	=			all_token_ids[i : i + self.seq_length]
  											if len(lowerCamelCase	) == self.seq_length:
  													self.current_size += 1
  													yield torch.tensor(lowerCamelCase	)
  
  
  
  
  
  					def      __a    (		self    :    Dict					,			lowerCamelCase    :    Optional[int]=10_00	):
  							return ShufflerIterDataPipe(self					,			buffer_size=lowerCamelCase	)
  
  
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__	):
  		lowerCamelCase_  :    Any   	=			args.model_ckpt.split('/'	)[-1]
  		lowerCamelCase_  :    List[Any]   	=			logging.getLogger(__name__	)
  		lowerCamelCase_  :    str   	=			Path(args.save_dir	) / 'log/'
  		log_dir.mkdir(exist_ok=lowerCAmelCase__	)
  		lowerCamelCase_  :    Dict   	=			F"debug_{accelerator.process_index}.log"
  		logging.basicConfig(
  		    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'       ,datefmt='%m/%d/%Y %H:%M:%S'       ,level=logging.INFO       ,handlers=[logging.FileHandler(log_dir / filename	), logging.StreamHandler()]       ,)
  		if accelerator.is_main_process:  # we only want to setup logging once
  				accelerator.init_trackers(lowerCAmelCase__       ,vars(lowerCAmelCase__	)	)
  				lowerCamelCase_  :    Optional[Any]   	=			accelerator.trackers[0].run.name
  				logger.setLevel(logging.INFO	)
  				datasets.utils.logging.set_verbosity_info()
  				transformers.utils.logging.set_verbosity_info()
  		else:
  				lowerCamelCase_  :    Optional[Any]   	=			''
  				logger.setLevel(logging.ERROR	)
  				datasets.utils.logging.set_verbosity_error()
  				transformers.utils.logging.set_verbosity_error()
  		return logger, run_name
  
  
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__	):
  		lowerCamelCase_  :    Optional[Any]   	=			{'streaming': True}
  		lowerCamelCase_  :    str   	=			load_dataset(args.dataset_name_train       ,split='train'       ,**lowerCAmelCase__	)
  		lowerCamelCase_  :    Any   	=			train_data.shuffle(buffer_size=args.shuffle_buffer       ,seed=args.seed	)
  		lowerCamelCase_  :    str   	=			load_dataset(args.dataset_name_valid       ,split='train'       ,**lowerCAmelCase__	)
  		lowerCamelCase_  :    Tuple   	=			ConstantLengthDataset(
  		    lowerCAmelCase__       ,lowerCAmelCase__       ,infinite=lowerCAmelCase__       ,seq_length=args.seq_length       ,tokenized=args.tokenized	)
  		lowerCamelCase_  :    str   	=			ConstantLengthDataset(
  		    lowerCAmelCase__       ,lowerCAmelCase__       ,infinite=lowerCAmelCase__       ,seq_length=args.seq_length       ,tokenized=args.tokenized	)
  		lowerCamelCase_  :    str   	=			train_dataset.shuffle(buffer_size=args.shuffle_buffer	)
  		lowerCamelCase_  :    List[Any]   	=			DataLoader(lowerCAmelCase__       ,batch_size=args.train_batch_size       ,shuffle=lowerCAmelCase__	)
  		lowerCamelCase_  :    List[Any]   	=			DataLoader(lowerCAmelCase__       ,batch_size=args.valid_batch_size	)
  		return train_dataloader, eval_dataloader
  
  
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__       ,lowerCAmelCase__       ,lowerCAmelCase__=["bias", "ln_1.weight", "ln_2.weight", "ln_f.weight"]	):
  		lowerCamelCase_	,			lowerCamelCase_  :    Union[str, Any]   	=			[], []
  		for n, p in model.named_parameters():
  				if any(nd in n for nd in no_decay	):
  						params_without_wd.append(lowerCAmelCase__	)
  				else:
  						params_with_wd.append(lowerCAmelCase__	)
  		return [
  		    {"params": params_with_wd, "weight_decay": args.weight_decay},
  		    {"params": params_without_wd, "weight_decay": 0.0},
  		]
  
  
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__       ,lowerCAmelCase__	):
  		logger.info(F"Step {step}: {metrics}"	)
  		if accelerator.is_main_process:
  				accelerator.log(lowerCAmelCase__       ,lowerCAmelCase__	)
  
  
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__       ,lowerCAmelCase__       ,lowerCAmelCase__	):
  		# TFLOPs formula (from Equation 3 in Section 5.1 of https://arxiv.org/pdf/2104.04473.pdf).
  		lowerCamelCase_  :    List[Any]   	=			accelerator.unwrap_model(lowerCAmelCase__	).config
  		lowerCamelCase_  :    Dict   	=			4 if args.gradient_checkpointing else 3
  		lowerCamelCase_  :    Any   	=			args.train_batch_size * accelerator.state.num_processes * args.gradient_accumulation_steps
  		lowerCamelCase_  :    List[Any]   	=			24 * checkpoint_factor * batch_size * args.seq_length * config_model.n_layer * (config_model.n_embd**2)
  		lowerCamelCase_  :    Union[str, Any]   	=			factor * (
  		    1.0
  		    + (args.seq_length / (6.0 * config_model.n_embd))
  		    + (tokenizer.vocab_size / (16.0 * config_model.n_layer * config_model.n_embd))
  		)
  		lowerCamelCase_  :    int   	=			flops_per_iteration / (elapsed_time * accelerator.state.num_processes * (10**12))
  		return tflops
  
  
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__	):
  		model.eval()
  		lowerCamelCase_  :    str   	=			[]
  		for step, batch in enumerate(lowerCAmelCase__	):
  				with torch.no_grad():
  						lowerCamelCase_  :    List[str]   	=			model(lowerCAmelCase__       ,labels=lowerCAmelCase__	)
  				lowerCamelCase_  :    int   	=			outputs.loss.repeat(args.valid_batch_size	)
  				losses.append(accelerator.gather(lowerCAmelCase__	)	)
  				if args.max_eval_steps > 0 and step >= args.max_eval_steps:
  						break
  		lowerCamelCase_  :    Dict   	=			torch.cat(lowerCAmelCase__	)
  		lowerCamelCase_  :    Dict   	=			losses[: eval_dataloader.dataset.current_size].mean()
  		try:
  				lowerCamelCase_  :    Optional[int]   	=			torch.exp(lowerCAmelCase__	)
  		except OverflowError:
  				lowerCamelCase_  :    Any   	=			float('inf'	)
  		return loss.item(), perplexity.item()
  
  
  # Settings
  _lowercase		:     Dict 					=HfArgumentParser(TrainingArguments)
  _lowercase		:     str 					=parser.parse_args()
  
  # Accelerator
  _lowercase		:     List[str] 					=ProjectConfiguration(project_dir=args.save_dir, logging_dir="""log""")
  _lowercase		:     Optional[int] 					=Accelerator(log_with=["""wandb""", """tensorboard"""], project_config=config)
  _lowercase		:     Any 					={str(k): str(v) for k, v in accelerator.state.__dict__.items()}
  
  _lowercase		:     str 					=Namespace(**vars(args), **acc_state)
  _lowercase		:     Any 					=accelerator.state.num_processes * args.train_batch_size
  set_seed(args.seed)
  
  # Clone model repository
  if accelerator.is_main_process:
  		_lowercase		:     Any 					=Repository(args.save_dir, clone_from=args.model_ckpt)
  
  # Logging
  _lowercase					,   _lowercase		:     Optional[int] 					=setup_logging(args)
  logger.info(accelerator.state)
  
  # Checkout new branch on repo
  if accelerator.is_main_process:
  		hf_repo.git_checkout(run_name, create_branch_ok=True)
  
  # Load model and tokenizer
  _lowercase		:     int 					=AutoModelForCausalLM.from_pretrained(args.save_dir)
  if args.gradient_checkpointing:
  		model.gradient_checkpointing_enable()
  _lowercase		:     Tuple 					=AutoTokenizer.from_pretrained(args.save_dir)
  
  # Load dataset and dataloader
  _lowercase					,   _lowercase		:     Dict 					=create_dataloaders(args)
  
  # Prepare the optimizer and learning rate scheduler
  _lowercase		:     List[str] 					=AdamW(get_grouped_params(model, args), lr=args.learning_rate)
  _lowercase		:     str 					=get_scheduler(
      name=args.lr_scheduler_type,
      optimizer=optimizer,
      num_warmup_steps=args.num_warmup_steps,
      num_training_steps=args.max_train_steps,
  )
  accelerator.register_for_checkpointing(lr_scheduler)
  
  
  def 		_SCREAMING_SNAKE_CASE    (   ):
  		return optimizer.param_groups[0]["lr"]
  
  
  # Prepare everything with our `accelerator`.
  _lowercase					,   _lowercase					,   _lowercase					,   _lowercase		:     str 					=accelerator.prepare(
      model, optimizer, train_dataloader, eval_dataloader
  )
  
  # load in the weights and states from a previous save
  if args.resume_from_checkpoint:
  		if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
  				accelerator.print(F'''Resumed from checkpoint: {args.resume_from_checkpoint}''')
  				accelerator.load_state(args.resume_from_checkpoint)
  				_lowercase		:     Optional[Any] 					=os.path.basename(args.resume_from_checkpoint)
  		else:
  				# Get the most recent checkpoint
  				_lowercase		:     int 					=[f.name for f in os.scandir(args.save_dir) if f.is_dir() and """step""" in str(f)]
  				dirs.sort(key=os.path.getctime)
  				_lowercase		:     Optional[Any] 					=dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
  		# Extract the step of the checkpoint to continue from there
  		_lowercase		:     Optional[int] 					=os.path.splitext(path)[0]
  		_lowercase		:     Union[str, Any] 					=int(training_difference.replace("""step_""", """"""))
  
  # Train model
  model.train()
  _lowercase		:     Dict 					=0
  _lowercase		:     Union[str, Any] 					=time.time()
  _lowercase		:     Dict 					=0
  for step, batch in enumerate(train_dataloader, start=1):
  		if args.resume_from_checkpoint and step < resume_step:
  				continue  # we need to skip steps until we reach the resumed step
  		_lowercase		:     int 					=model(batch, labels=batch, use_cache=False).loss
  		_lowercase		:     int 					=accelerator.gather(loss.repeat(args.train_batch_size)).mean()
  		loss_tracking += avg_loss.item() / args.gradient_accumulation_steps
  		log_metrics(step, {"""samples""": step * samples_per_step, """loss_per_step/train""": loss.item()})
  		_lowercase		:     Any 					=loss / args.gradient_accumulation_steps
  		if step % args.gradient_accumulation_steps != 0:
  				# Prevent backward from doing gradient all_reduce in every step
  				if accelerator.distributed_type == DistributedType.MULTI_GPU:
  						with model.no_sync():
  								accelerator.backward(loss)
  				else:
  						accelerator.backward(loss)
  		else:
  				_lowercase		:     Tuple 					=get_lr()
  				accelerator.backward(loss)
  				accelerator.clip_grad_norm_(model.parameters(), 1.0)
  				optimizer.step()
  				lr_scheduler.step()
  				optimizer.zero_grad()
  				_lowercase		:     Dict 					=time.time() - t_start
  				_lowercase		:     Union[str, Any] 					=compute_tflops(elapsed_time, accelerator, args)
  				log_metrics(
  				    step,
  				    {
  				        """steps""": completed_steps,
  				        """loss/train""": loss_tracking,
  				        """lr""": lr,
  				        """tflops""": tflops,
  				        """time_per_iteration""": elapsed_time,
  				    },
  				)
  				_lowercase		:     Tuple 					=time.time()
  				_lowercase		:     Dict 					=0
  				completed_steps += 1
  		if step % args.save_checkpoint_steps == 0:
  				logger.info("""Evaluating and saving model checkpoint""")
  				_lowercase					,   _lowercase		:     Dict 					=evaluate(args)
  				log_metrics(step, {"""loss/eval""": eval_loss, """perplexity""": perplexity})
  				accelerator.wait_for_everyone()
  				_lowercase		:     Optional[Any] 					=os.path.join(args.save_dir, F'''step_{step}''')
  				accelerator.save_state(save_dir)
  				if accelerator.is_main_process:
  						hf_repo.push_to_hub(commit_message=F'''step {step}''')
  				model.train()
  		if completed_steps >= args.max_train_steps:
  				break
  
  # Evaluate and save the last checkpoint
  logger.info("""Evaluating and saving model after training""")
  _lowercase					,   _lowercase		:     Union[str, Any] 					=evaluate(args)
  log_metrics(step, {"""loss/eval""": eval_loss, """perplexity""": perplexity})
  accelerator.wait_for_everyone()
  _lowercase		:     str 					=accelerator.unwrap_model(model)
  unwrapped_model.save_pretrained(args.save_dir, save_function=accelerator.save)
  _lowercase		:     Tuple 					=os.path.join(args.save_dir, F'''step_{step}''')
  accelerator.save_state(save_dir)
  if accelerator.is_main_process:
  		hf_repo.push_to_hub(commit_message="""final model""")
  
style_context: 
  
  
  
  
  
  from typing import Optional, Tuple, Union
  
  import tensorflow as tf
  
  from ...activations_tf import ACTaFN
  from ...modeling_tf_outputs import (
      TFBaseModelOutputWithNoAttention,
      TFBaseModelOutputWithPoolingAndNoAttention,
      TFImageClassifierOutputWithNoAttention,
  )
  from ...modeling_tf_utils import TFPreTrainedModel, TFSequenceClassificationLoss, keras_serializable, unpack_inputs
  from ...tf_utils import shape_list
  from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
  from .configuration_resnet import ResNetConfig
  
  
  _lowercase		:     Optional[Any] 					=logging.get_logger(__name__)
  
  # General docstring
  _lowercase		:     Union[str, Any] 					="""ResNetConfig"""
  
  # Base docstring
  _lowercase		:     Dict 					="""microsoft/resnet-50"""
  _lowercase		:     Tuple 					=[1, 2048, 7, 7]
  
  # Image classification docstring
  _lowercase		:     Dict 					="""microsoft/resnet-50"""
  _lowercase		:     int 					="""tiger cat"""
  
  _lowercase		:     Optional[Any] 					=[
      """microsoft/resnet-50""",
      # See all resnet models at https://huggingface.co/models?filter=resnet
  ]
  
  
  
  
  
  
  class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):
  
  
  
  					def __init__(		self    :    List[str]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 3					,			lowerCamelCase    :    int = 1					,			lowerCamelCase    :    str = "relu"					,			**lowerCamelCase    :    Tuple	):
  							super().__init__(**lowerCamelCase	)
  							lowerCamelCase_  :    Tuple   	=			kernel_size // 2
  							lowerCamelCase_  :    Optional[int]   	=			tf.keras.layers.ConvaD(
  							    lowerCamelCase					,			kernel_size=lowerCamelCase					,			strides=lowerCamelCase					,			padding='valid'					,			use_bias=lowerCamelCase					,			name='convolution'	)
  							# Use same default momentum and epsilon as PyTorch equivalent
  							lowerCamelCase_  :    Union[str, Any]   	=			tf.keras.layers.BatchNormalization(epsilon=1E-5					,			momentum=0.9					,			name='normalization'	)
  							lowerCamelCase_  :    Optional[Any]   	=			ACTaFN[activation] if activation is not None else tf.keras.layers.Activation('linear'	)
  
  
  
  					def      __a    (		self    :    int					,			lowerCamelCase    :    tf.Tensor	):
  							# Pad to match that done in the PyTorch Conv2D model
  							lowerCamelCase_  :    int   	=			(self.pad_value, self.pad_value)
  							lowerCamelCase_  :    Optional[Any]   	=			tf.pad(lowerCamelCase					,			[(0, 0), height_pad, width_pad, (0, 0)]	)
  							lowerCamelCase_  :    str   	=			self.conv(lowerCamelCase	)
  							return hidden_state
  
  
  
  
  
  					def      __a    (		self    :    Tuple					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
  							lowerCamelCase_  :    Optional[Any]   	=			self.convolution(lowerCamelCase	)
  							lowerCamelCase_  :    int   	=			self.normalization(lowerCamelCase					,			training=lowerCamelCase	)
  							lowerCamelCase_  :    Any   	=			self.activation(lowerCamelCase	)
  							return hidden_state
  
  
  
  
  
  
  class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):
  
  
  
  					def __init__(		self    :    Dict					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    Optional[Any]	):
  							super().__init__(**lowerCamelCase	)
  							lowerCamelCase_  :    Union[str, Any]   	=			TFResNetConvLayer(
  							    config.embedding_size					,			kernel_size=7					,			stride=2					,			activation=config.hidden_act					,			name='embedder'					,			)
  							lowerCamelCase_  :    Any   	=			tf.keras.layers.MaxPoolaD(pool_size=3					,			strides=2					,			padding='valid'					,			name='pooler'	)
  							lowerCamelCase_  :    Any   	=			config.num_channels
  
  
  
  
  
  					def      __a    (		self    :    List[Any]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
  							lowerCamelCase_	,			lowerCamelCase_	,			lowerCamelCase_	,			lowerCamelCase_  :    Tuple   	=			shape_list(lowerCamelCase	)
  							if tf.executing_eagerly() and num_channels != self.num_channels:
  									raise ValueError(
  									    'Make sure that the channel dimension of the pixel values match with the one set in the configuration.'	)
  							lowerCamelCase_  :    Optional[int]   	=			pixel_values
  							lowerCamelCase_  :    Optional[Any]   	=			self.embedder(lowerCamelCase	)
  							lowerCamelCase_  :    Optional[int]   	=			tf.pad(lowerCamelCase					,			[[0, 0], [1, 1], [1, 1], [0, 0]]	)
  							lowerCamelCase_  :    str   	=			self.pooler(lowerCamelCase	)
  							return hidden_state
  
  
  
  
  
  
  class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):
  
  
  
  					def __init__(		self    :    Union[str, Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 2					,			**lowerCamelCase    :    List[Any]	):
  							super().__init__(**lowerCamelCase	)
  							lowerCamelCase_  :    int   	=			tf.keras.layers.ConvaD(
  							    lowerCamelCase					,			kernel_size=1					,			strides=lowerCamelCase					,			use_bias=lowerCamelCase					,			name='convolution'	)
  							# Use same default momentum and epsilon as PyTorch equivalent
  							lowerCamelCase_  :    Any   	=			tf.keras.layers.BatchNormalization(epsilon=1E-5					,			momentum=0.9					,			name='normalization'	)
  
  
  
  
  
  					def      __a    (		self    :    Tuple					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
  							lowerCamelCase_  :    Union[str, Any]   	=			x
  							lowerCamelCase_  :    Optional[int]   	=			self.convolution(lowerCamelCase	)
  							lowerCamelCase_  :    List[str]   	=			self.normalization(lowerCamelCase					,			training=lowerCamelCase	)
  							return hidden_state
  
  
  
  
  
  
  class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):
  
  
  
  					def __init__(		self    :    List[Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 1					,			lowerCamelCase    :    str = "relu"					,			**lowerCamelCase    :    Optional[int]	):
  							super().__init__(**lowerCamelCase	)
  							lowerCamelCase_  :    Any   	=			in_channels != out_channels or stride != 1
  							lowerCamelCase_  :    Tuple   	=			TFResNetConvLayer(lowerCamelCase					,			stride=lowerCamelCase					,			name='layer.0'	)
  							lowerCamelCase_  :    str   	=			TFResNetConvLayer(lowerCamelCase					,			activation=lowerCamelCase					,			name='layer.1'	)
  							lowerCamelCase_  :    Any   	=			(
  							    TFResNetShortCut(lowerCamelCase					,			stride=lowerCamelCase					,			name='shortcut'	)
  							    if should_apply_shortcut
  							    else tf.keras.layers.Activation('linear'					,			name='shortcut'	)
  							)
  							lowerCamelCase_  :    str   	=			ACTaFN[activation]
  
  
  
  
  
  					def      __a    (		self    :    List[Any]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
  							lowerCamelCase_  :    Optional[int]   	=			hidden_state
  							lowerCamelCase_  :    Tuple   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
  							lowerCamelCase_  :    List[Any]   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
  							lowerCamelCase_  :    int   	=			self.shortcut(lowerCamelCase					,			training=lowerCamelCase	)
  							hidden_state += residual
  							lowerCamelCase_  :    Dict   	=			self.activation(lowerCamelCase	)
  							return hidden_state
  
  
  
  
  
  
  class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):
  
  
  
  					def __init__(		self    :    Optional[Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 1					,			lowerCamelCase    :    str = "relu"					,			lowerCamelCase    :    int = 4					,			**lowerCamelCase    :    Optional[int]					,			):
  							super().__init__(**lowerCamelCase	)
  							lowerCamelCase_  :    List[Any]   	=			in_channels != out_channels or stride != 1
  							lowerCamelCase_  :    Optional[Any]   	=			out_channels // reduction
  							lowerCamelCase_  :    int   	=			TFResNetConvLayer(lowerCamelCase					,			kernel_size=1					,			name='layer.0'	)
  							lowerCamelCase_  :    Tuple   	=			TFResNetConvLayer(lowerCamelCase					,			stride=lowerCamelCase					,			name='layer.1'	)
  							lowerCamelCase_  :    Optional[int]   	=			TFResNetConvLayer(lowerCamelCase					,			kernel_size=1					,			activation=lowerCamelCase					,			name='layer.2'	)
  							lowerCamelCase_  :    Optional[Any]   	=			(
  							    TFResNetShortCut(lowerCamelCase					,			stride=lowerCamelCase					,			name='shortcut'	)
  							    if should_apply_shortcut
  							    else tf.keras.layers.Activation('linear'					,			name='shortcut'	)
  							)
  							lowerCamelCase_  :    List[Any]   	=			ACTaFN[activation]
  
  
  
  
  
  					def      __a    (		self    :    str					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
  							lowerCamelCase_  :    int   	=			hidden_state
  							lowerCamelCase_  :    Tuple   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
  							lowerCamelCase_  :    List[Any]   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
  							lowerCamelCase_  :    List[str]   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
  							lowerCamelCase_  :    List[Any]   	=			self.shortcut(lowerCamelCase					,			training=lowerCamelCase	)
  							hidden_state += residual
  							lowerCamelCase_  :    int   	=			self.activation(lowerCamelCase	)
  							return hidden_state
  
  
  
  
  
  
  class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):
  
  
  
  					def __init__(		self    :    int					,			lowerCamelCase    :    ResNetConfig					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 2					,			lowerCamelCase    :    int = 2					,			**lowerCamelCase    :    List[Any]	):
  							super().__init__(**lowerCamelCase	)
  
  							lowerCamelCase_  :    List[Any]   	=			TFResNetBottleNeckLayer if config.layer_type == 'bottleneck' else TFResNetBasicLayer
  
  							lowerCamelCase_  :    Tuple   	=			[layer(lowerCamelCase					,			lowerCamelCase					,			stride=lowerCamelCase					,			activation=config.hidden_act					,			name='layers.0'	)]
  							layers += [
  							    layer(lowerCamelCase					,			lowerCamelCase					,			activation=config.hidden_act					,			name=F"layers.{i + 1}"	)
  							    for i in range(depth - 1	)
  							]
  							lowerCamelCase_  :    Dict   	=			layers
  
  
  
  
  
  					def      __a    (		self    :    Any					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
  							for layer in self.stage_layers:
  									lowerCamelCase_  :    Any   	=			layer(lowerCamelCase					,			training=lowerCamelCase	)
  							return hidden_state
  
  
  
  
  
  
  class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):
  
  
  
  					def __init__(		self    :    List[Any]					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    Optional[Any]	):
  							super().__init__(**lowerCamelCase	)
  							# based on `downsample_in_first_stage` the first layer of the first stage may or may not downsample the input
  							lowerCamelCase_  :    List[Any]   	=			[
  							    TFResNetStage(
  							        lowerCamelCase					,			config.embedding_size					,			config.hidden_sizes[0]					,			stride=2 if config.downsample_in_first_stage else 1					,			depth=config.depths[0]					,			name='stages.0'					,			)
  							]
  							for i, (in_channels, out_channels, depth) in enumerate(
  							    zip(config.hidden_sizes					,			config.hidden_sizes[1:]					,			config.depths[1:]	)	):
  									self.stages.append(TFResNetStage(lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			depth=lowerCamelCase					,			name=F"stages.{i + 1}"	)	)
  
  
  
  
  
  					def      __a    (		self    :    Optional[int]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False					,			lowerCamelCase    :    bool = True					,			lowerCamelCase    :    bool = False					,			):
  							lowerCamelCase_  :    Optional[int]   	=			() if output_hidden_states else None
  
  							for stage_module in self.stages:
  									if output_hidden_states:
  											lowerCamelCase_  :    Any   	=			hidden_states + (hidden_state,)
  
  									lowerCamelCase_  :    Optional[Any]   	=			stage_module(lowerCamelCase					,			training=lowerCamelCase	)
  
  							if output_hidden_states:
  									lowerCamelCase_  :    Union[str, Any]   	=			hidden_states + (hidden_state,)
  
  							if not return_dict:
  									return tuple(v for v in [hidden_state, hidden_states] if v is not None	)
  
  							return TFBaseModelOutputWithNoAttention(last_hidden_state=lowerCamelCase					,			hidden_states=lowerCamelCase	)
  
  
  
  
  
  
  class 	UpperCamelCase_     (     snake_case__       ):
  					_a			:	int							=				ResNetConfig
  					_a			:	Optional[Any]							=				'resnet'
  					_a			:	int							=				'pixel_values'
  
  
  
  					@property
  					def      __a    (		self    :    Tuple	):
  							return {"pixel_values": tf.TensorSpec(shape=(None, self.config.num_channels, 2_24, 2_24)					,			dtype=tf.floataa	)}
  _lowercase		:     Union[str, Any] 					=r"""
      This model is a TensorFlow
      [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) sub-class. Use it as a
      regular TensorFlow Module and refer to the TensorFlow documentation for all matter related to general usage and
      behavior.
  
      Parameters:
          config ([`ResNetConfig`]): Model configuration class with all the parameters of the model.
              Initializing with a config file does not load the weights associated with the model, only the
              configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
  """
  
  
  _lowercase		:     Optional[Any] 					=r"""
      Args:
          pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):
              Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
              [`ConvNextImageProcessor.__call__`] for details.
  
          output_hidden_states (`bool`, *optional*):
              Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
              more detail.
          return_dict (`bool`, *optional*):
              Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
  """
  
  
  
  
  
  
  @keras_serializable
  class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):
  					_a			:	int							=				ResNetConfig
  
  
  
  					def __init__(		self    :    Dict					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    List[str]	):
  							super().__init__(**lowerCamelCase	)
  							lowerCamelCase_  :    Tuple   	=			config
  							lowerCamelCase_  :    int   	=			TFResNetEmbeddings(lowerCamelCase					,			name='embedder'	)
  							lowerCamelCase_  :    Union[str, Any]   	=			TFResNetEncoder(lowerCamelCase					,			name='encoder'	)
  							lowerCamelCase_  :    Tuple   	=			tf.keras.layers.GlobalAveragePoolingaD(keepdims=lowerCamelCase	)
  
  
  
  
  
  					@unpack_inputs
  					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    Optional[bool] = None					,			lowerCamelCase    :    Optional[bool] = None					,			lowerCamelCase    :    bool = False					,			):
  							lowerCamelCase_  :    str   	=			(
  							    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
  							)
  							lowerCamelCase_  :    Optional[Any]   	=			return_dict if return_dict is not None else self.config.use_return_dict
  
  							# TF 2.0 image layers can't use NCHW format when running on CPU.
  							# We transpose to NHWC format and then transpose back after the full forward pass.
  							# (batch_size, num_channels, height, width) -> (batch_size, height, width, num_channels)
  							lowerCamelCase_  :    str   	=			tf.transpose(lowerCamelCase					,			perm=[0, 2, 3, 1]	)
  							lowerCamelCase_  :    Any   	=			self.embedder(lowerCamelCase					,			training=lowerCamelCase	)
  
  							lowerCamelCase_  :    str   	=			self.encoder(
  							    lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			return_dict=lowerCamelCase					,			training=lowerCamelCase	)
  
  							lowerCamelCase_  :    Optional[int]   	=			encoder_outputs[0]
  
  							lowerCamelCase_  :    str   	=			self.pooler(lowerCamelCase	)
  
  							# Transpose all the outputs to the NCHW format
  							# (batch_size, height, width, num_channels) -> (batch_size, num_channels, height, width)
  							lowerCamelCase_  :    str   	=			tf.transpose(lowerCamelCase					,			(0, 3, 1, 2)	)
  							lowerCamelCase_  :    Tuple   	=			tf.transpose(lowerCamelCase					,			(0, 3, 1, 2)	)
  							lowerCamelCase_  :    Dict   	=			()
  							for hidden_state in encoder_outputs[1:]:
  									lowerCamelCase_  :    Dict   	=			hidden_states + tuple(tf.transpose(lowerCamelCase					,			(0, 3, 1, 2)	) for h in hidden_state	)
  
  							if not return_dict:
  									return (last_hidden_state, pooled_output) + hidden_states
  
  							lowerCamelCase_  :    Dict   	=			hidden_states if output_hidden_states else None
  
  							return TFBaseModelOutputWithPoolingAndNoAttention(
  							    last_hidden_state=lowerCamelCase					,			pooler_output=lowerCamelCase					,			hidden_states=lowerCamelCase					,			)
  
  
  
  
  
  
  @add_start_docstrings(
      'The bare ResNet model outputting raw features without any specific head on top.'     ,    snake_case__     ,    )
  class 	UpperCamelCase_     (     snake_case__       ):
  
  
  
  					def __init__(		self    :    Any					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    int	):
  							super().__init__(lowerCamelCase					,			**lowerCamelCase	)
  							lowerCamelCase_  :    List[str]   	=			TFResNetMainLayer(config=lowerCamelCase					,			name='resnet'	)
  
  
  
  
  
  					@add_start_docstrings_to_model_forward(lowerCamelCase	)
  					@add_code_sample_docstrings(
  					    checkpoint=_CHECKPOINT_FOR_DOC					,			output_type=lowerCamelCase					,			config_class=_CONFIG_FOR_DOC					,			modality='vision'					,			expected_output=_EXPECTED_OUTPUT_SHAPE					,			)
  					@unpack_inputs
  					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    Optional[bool] = None					,			lowerCamelCase    :    Optional[bool] = None					,			lowerCamelCase    :    bool = False					,			):
  							lowerCamelCase_  :    List[str]   	=			(
  							    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
  							)
  							lowerCamelCase_  :    Tuple   	=			return_dict if return_dict is not None else self.config.use_return_dict
  
  							lowerCamelCase_  :    Dict   	=			self.resnet(
  							    pixel_values=lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			return_dict=lowerCamelCase					,			training=lowerCamelCase					,			)
  							return resnet_outputs
  
  
  
  
  
  
  @add_start_docstrings(
      '\n    ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    '     ,    snake_case__     ,    )
  class 	UpperCamelCase_     (     snake_case__     ,    snake_case__       ):
  
  
  
  					def __init__(		self    :    Tuple					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    Optional[Any]	):
  							super().__init__(lowerCamelCase					,			**lowerCamelCase	)
  							lowerCamelCase_  :    Tuple   	=			config.num_labels
  							lowerCamelCase_  :    Union[str, Any]   	=			TFResNetMainLayer(lowerCamelCase					,			name='resnet'	)
  							# classification head
  							lowerCamelCase_  :    Dict   	=			(
  							    tf.keras.layers.Dense(config.num_labels					,			name='classifier.1'	)
  							    if config.num_labels > 0
  							    else tf.keras.layers.Activation('linear'					,			name='classifier.1'	)
  							)
  
  
  
  					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    tf.Tensor	):
  							lowerCamelCase_  :    Tuple   	=			tf.keras.layers.Flatten()(lowerCamelCase	)
  							lowerCamelCase_  :    Dict   	=			self.classifier_layer(lowerCamelCase	)
  							return logits
  
  
  
  
  
  					@add_start_docstrings_to_model_forward(lowerCamelCase	)
  					@add_code_sample_docstrings(
  					    checkpoint=_IMAGE_CLASS_CHECKPOINT					,			output_type=lowerCamelCase					,			config_class=_CONFIG_FOR_DOC					,			expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT					,			)
  					@unpack_inputs
  					def      __a    (		self    :    Optional[int]					,			lowerCamelCase    :    tf.Tensor = None					,			lowerCamelCase    :    tf.Tensor = None					,			lowerCamelCase    :    bool = None					,			lowerCamelCase    :    bool = None					,			lowerCamelCase    :    bool = False					,			):
  							lowerCamelCase_  :    List[str]   	=			return_dict if return_dict is not None else self.config.use_return_dict
  
  							lowerCamelCase_  :    Optional[int]   	=			self.resnet(
  							    lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			return_dict=lowerCamelCase					,			training=lowerCamelCase	)
  
  							lowerCamelCase_  :    Union[str, Any]   	=			outputs.pooler_output if return_dict else outputs[1]
  
  							lowerCamelCase_  :    Union[str, Any]   	=			self.classifier(lowerCamelCase	)
  
  							lowerCamelCase_  :    List[Any]   	=			None if labels is None else self.hf_compute_loss(lowerCamelCase					,			lowerCamelCase	)
  
  							if not return_dict:
  									lowerCamelCase_  :    int   	=			(logits,) + outputs[2:]
  									return (loss,) + output if loss is not None else output
  
  							return TFImageClassifierOutputWithNoAttention(loss=lowerCamelCase					,			logits=lowerCamelCase					,			hidden_states=outputs.hidden_states	)
  