code: 
  # This test is meant to be run in on an instance with TPUs like this:
  #
  #   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
  #
  # Replace 8 with the number of TPU cores you have.
  #
  
  import sys
  from typing import Dict
  
  from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
  from transformers.utils import logging
  
  
  _snake_case			:       Optional[Any]      =						logging.get_logger(__name__)
  
  
  if is_torch_available():
  						import torch
  						from torch import nn
  						from torch.utils.data import Dataset
  
  						from transformers import Trainer
  
  
  
  
  
  						class      UpperCamelCase_			(			__a		):
  
  
  
  
  
  
  
  												'''simple docstring'''
  
  
  
  												def __init__( self       :Optional[int]		,						lowerCAmelCase__       :int = 101    )     ->Any:
  																lowercase						    =			length
  
  
  
  
  
  												def __len__( self       :Tuple    )     ->Any:
  																return self.length
  
  
  
  
  
  												def __getitem__( self       :Any		,						lowerCAmelCase__       :Tuple    )     ->int:
  																return i
  
  
  
  
  
  						class      UpperCamelCase_			:
  
  
  
  
  
  
  
  												'''simple docstring'''
  
  
  
  												def __call__( self       :Optional[Any]		,						lowerCAmelCase__       :List[Any]    )     ->int:
  																return {"input_ids": torch.tensor(lowerCAmelCase__    ), "labels": torch.tensor(lowerCAmelCase__    )}
  
  
  
  
  
  						class      UpperCamelCase_			(			nn.Module		):
  
  
  
  
  
  
  
  												'''simple docstring'''
  
  
  
  												def __init__( self       :Any    )     ->str:
  																super().__init__()
  																# Add some (unused) params otherwise DDP will complain.
  																lowercase						    =			nn.Linear(120		,						80    )
  
  
  
  
  
  												def 					SCREAMING_SNAKE_CASE( self       :int		,						lowerCAmelCase__       :Optional[int]		,						lowerCAmelCase__       :Union[str, Any]=None    )     ->List[Any]:
  																if labels is not None:
  																				return torch.tensor(0.0		,						device=input_ids.device    ), input_ids
  																else:
  																				return input_ids
  
  def    __snake_case       (  ):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			HfArgumentParser((TrainingArguments,)	)
  				sys.argv += ["--output_dir", "./examples"]
  				lowercase						    =			parser.parse_args_into_dataclasses()[0]
  
  				logger.warning(
  				    F'''Process rank: {training_args.local_rank}, device: {training_args.device}, '''
  				    F'''tpu_num_cores: {training_args.tpu_num_cores}'''		,						)
  
  				# Essentially, what we want to verify in the distributed case is
  				# that we get all samples back, in the right order.
  				# (this is crucial for prediction for instance)
  				for dataset_length in [1001, 256, 15]:
  								lowercase						    =			DummyDataset(__magic_name__	)
  
  								def compute_metrics(__magic_name__	) -> Dict:
  												lowercase						    =			list(range(len(__magic_name__	)	)	)
  												lowercase						    =			p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
  												return {"success": success}
  
  								lowercase						    =			Trainer(
  								    model=DummyModel()		,						args=__magic_name__		,						data_collator=DummyDataCollator()		,						eval_dataset=__magic_name__		,						compute_metrics=__magic_name__		,						)
  								lowercase						    =			trainer.evaluate()
  								logger.info(__magic_name__	)
  								if metrics["eval_success"] is not True:
  												logger.error(__magic_name__	)
  												exit(1	)
  
  								lowercase						    =			trainer.predict(__magic_name__	)
  								logger.info(p.metrics	)
  								if p.metrics["test_success"] is not True:
  												logger.error(p.metrics	)
  												exit(1	)
  
  								lowercase						    =			2
  
  								lowercase						    =			trainer.evaluate()
  								logger.info(__magic_name__	)
  								if metrics["eval_success"] is not True:
  												logger.error(__magic_name__	)
  												exit(1	)
  
  								lowercase						    =			trainer.predict(__magic_name__	)
  								logger.info(p.metrics	)
  								if p.metrics["test_success"] is not True:
  												logger.error(p.metrics	)
  												exit(1	)
  
  								lowercase						    =			None
  
  				logger.info("🔥 All distributed tests successful"	)
  
  def    __snake_case       (  __magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				main()
  
  
  if __name__ == "__main__":
  						main()
  
style_context: 
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  _snake_case			:       List[Any]      =						logging.get_logger(__name__)
  
  _snake_case			:       Dict      =						{
      "google/reformer-crime-and-punishment": (
          "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/config.json"
      ),
      "google/reformer-enwik8": "https://huggingface.co/google/reformer-enwik8/resolve/main/config.json",
  }
  
  
  
  
  
  class      UpperCamelCase_			(			__a		):
  
  
  
  
  
  
  
  						'''simple docstring'''
  
  
  
  						UpperCamelCase			:     Any							      =				'''reformer'''
  						UpperCamelCase			:     List[str]							      =				['''past_buckets_states''']
  						UpperCamelCase			:     Optional[int]							      =				{}
  
  
  
  
  
  						def __init__( self       :Dict		,						lowerCAmelCase__       :Any=64		,						lowerCAmelCase__       :str=["local", "lsh", "local", "lsh", "local", "lsh"]		,						lowerCAmelCase__       :Optional[Any]=1.0		,						lowerCAmelCase__       :List[str]=True		,						lowerCAmelCase__       :Optional[Any]=[64, 64]		,						lowerCAmelCase__       :Tuple=[64, 192]		,						lowerCAmelCase__       :Union[str, Any]=0		,						lowerCAmelCase__       :Optional[Any]=2		,						lowerCAmelCase__       :List[str]=512		,						lowerCAmelCase__       :Any=None		,						lowerCAmelCase__       :int="relu"		,						lowerCAmelCase__       :Tuple=0.05		,						lowerCAmelCase__       :int=256		,						lowerCAmelCase__       :Optional[Any]=0.02		,						lowerCAmelCase__       :Optional[int]=False		,						lowerCAmelCase__       :Optional[Any]=1E-1_2		,						lowerCAmelCase__       :int=1		,						lowerCAmelCase__       :Optional[Any]=0		,						lowerCAmelCase__       :Any=0.05		,						lowerCAmelCase__       :Optional[int]=64		,						lowerCAmelCase__       :List[Any]=64		,						lowerCAmelCase__       :Tuple=0.0		,						lowerCAmelCase__       :Tuple=1		,						lowerCAmelCase__       :List[str]=0		,						lowerCAmelCase__       :Dict=4096		,						lowerCAmelCase__       :Tuple=12		,						lowerCAmelCase__       :Tuple=None		,						lowerCAmelCase__       :Union[str, Any]=1		,						lowerCAmelCase__       :int=0		,						lowerCAmelCase__       :Optional[Any]=320		,						lowerCAmelCase__       :Optional[Any]=False		,						lowerCAmelCase__       :Optional[int]=True		,						lowerCAmelCase__       :str=None		,						**lowerCAmelCase__       :Any		,						)     ->Any:
  										lowercase						    =			hash_seed
  										lowercase						    =			vocab_size
  										lowercase						    =			attention_head_size
  										lowercase						    =			hidden_size
  										lowercase						    =			num_attention_heads
  										lowercase						    =			num_hashes
  										lowercase						    =			len(lowerCAmelCase__    )
  										lowercase						    =			tuple(lowerCAmelCase__    ) if isinstance(lowerCAmelCase__		,						lowerCAmelCase__    ) else num_buckets
  										lowercase						    =			lsh_attn_chunk_length
  										lowercase						    =			local_attn_chunk_length
  										lowercase						    =			lsh_num_chunks_after
  										lowercase						    =			lsh_num_chunks_before
  										lowercase						    =			local_num_chunks_after
  										lowercase						    =			local_num_chunks_before
  										lowercase						    =			hidden_act
  										lowercase						    =			feed_forward_size
  										lowercase						    =			hidden_dropout_prob
  										lowercase						    =			lsh_attention_probs_dropout_prob
  										lowercase						    =			local_attention_probs_dropout_prob
  										lowercase						    =			max_position_embeddings
  										lowercase						    =			initializer_range
  										lowercase						    =			layer_norm_eps
  										lowercase						    =			axial_pos_embds
  										lowercase						    =			tuple(lowerCAmelCase__    )
  										lowercase						    =			tuple(lowerCAmelCase__    )
  										lowercase						    =			axial_norm_std
  										lowercase						    =			chunk_size_lm_head
  										lowercase						    =			attn_layers
  										lowercase						    =			use_cache
  										lowercase						    =			classifier_dropout
  										super().__init__(
  										    pad_token_id=lowerCAmelCase__		,						eos_token_id=lowerCAmelCase__		,						is_decoder=lowerCAmelCase__		,						tie_word_embeddings=lowerCAmelCase__		,						**lowerCAmelCase__		,						)
  