
import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu







class        A_   (			nn.Module   ):




	'''simple docstring'''


	def __init__(       self       ):
		super().__init__()
		lowercase       =    nn.Linear(3		,    4       )
		lowercase       =    nn.BatchNormad(4       )
		lowercase       =    nn.Linear(4		,    5       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		return self.lineara(self.batchnorm(self.lineara(snake_case       )       )       )







class        A_   (			__lowerCamelCase   ):




	'''simple docstring'''


	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    *snake_case		,    **snake_case       ):
		return (args[0] + 1,) + args[1:], kwargs







class        A_   (			__lowerCamelCase   ):




	'''simple docstring'''


	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case       ):
		return output + 1







class        A_   (			unittest.TestCase   ):




	'''simple docstring'''


	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ModelForTest()
		lowercase       =    ModelHook()

		add_hook_to_module(snake_case		,    snake_case       )
		self.assertEqual(test_model._hf_hook		,    snake_case       )
		self.assertTrue(hasattr(snake_case		,    '_old_forward'       )       )

		# Check adding the hook did not change the name or the signature
		self.assertEqual(test_model.forward.__name__		,    'forward'       )
		self.assertListEqual(list(inspect.signature(test_model.forward       ).parameters       )		,    ['x']       )

		remove_hook_from_module(snake_case       )
		self.assertFalse(hasattr(snake_case		,    '_hf_hook'       )       )
		self.assertFalse(hasattr(snake_case		,    '_old_forward'       )       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ModelForTest()
		lowercase       =    ModelHook()

		add_hook_to_module(snake_case		,    snake_case       )
		add_hook_to_module(snake_case		,    snake_case		,    append=snake_case       )

		self.assertEqual(isinstance(test_model._hf_hook		,    snake_case       )		,    snake_case       )
		self.assertEqual(len(test_model._hf_hook.hooks       )		,    2       )
		self.assertTrue(hasattr(snake_case		,    '_old_forward'       )       )

		# Check adding the hook did not change the name or the signature
		self.assertEqual(test_model.forward.__name__		,    'forward'       )
		self.assertListEqual(list(inspect.signature(test_model.forward       ).parameters       )		,    ['x']       )

		remove_hook_from_module(snake_case       )
		self.assertFalse(hasattr(snake_case		,    '_hf_hook'       )       )
		self.assertFalse(hasattr(snake_case		,    '_old_forward'       )       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ModelForTest()
		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    test_model(x + 1       )
		lowercase       =    test_model(x + 2       )

		lowercase       =    PreForwardHook()
		add_hook_to_module(snake_case		,    snake_case       )
		lowercase       =    test_model(snake_case       )
		self.assertTrue(torch.allclose(snake_case		,    snake_case		,    atol=1E-5       )       )

		# Attaching a hook to a model when it already has one replaces, does not chain
		lowercase       =    PreForwardHook()
		add_hook_to_module(snake_case		,    snake_case       )
		lowercase       =    test_model(snake_case       )
		self.assertTrue(torch.allclose(snake_case		,    snake_case		,    atol=1E-5       )       )

		# You need to use the sequential hook to chain two or more hooks
		lowercase       =    SequentialHook(PreForwardHook()		,    PreForwardHook()       )
		add_hook_to_module(snake_case		,    snake_case       )

		lowercase       =    test_model(snake_case       )
		assert torch.allclose(snake_case		,    snake_case		,    atol=1E-5       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ModelForTest()
		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    test_model(snake_case       )

		lowercase       =    PostForwardHook()
		add_hook_to_module(snake_case		,    snake_case       )
		lowercase       =    test_model(snake_case       )
		self.assertTrue(torch.allclose(snake_case		,    output + 1		,    atol=1E-5       )       )

		# Attaching a hook to a model when it already has one replaces, does not chain
		lowercase       =    PostForwardHook()
		add_hook_to_module(snake_case		,    snake_case       )
		lowercase       =    test_model(snake_case       )
		self.assertTrue(torch.allclose(snake_case		,    output + 1		,    atol=1E-5       )       )

		# You need to use the sequential hook to chain two or more hooks
		lowercase       =    SequentialHook(PostForwardHook()		,    PostForwardHook()       )
		add_hook_to_module(snake_case		,    snake_case       )

		lowercase       =    test_model(snake_case       )
		assert torch.allclose(snake_case		,    output + 2		,    atol=1E-5       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ModelForTest()
		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    test_model(snake_case       )

		lowercase       =    PostForwardHook()
		add_hook_to_module(snake_case		,    snake_case       )
		lowercase       =    test_model(snake_case       )
		self.assertTrue(torch.allclose(snake_case		,    output + 1       )       )
		self.assertTrue(outputa.requires_grad       )

		lowercase       =    True
		lowercase       =    test_model(snake_case       )
		self.assertFalse(outputa.requires_grad       )





	@require_multi_gpu
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ModelForTest()
		# Everything is on CPU
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )

		# This will move each submodule on different devices
		add_hook_to_module(model.lineara		,    AlignDevicesHook(execution_device=0       )       )
		add_hook_to_module(model.batchnorm		,    AlignDevicesHook(execution_device=0       )       )
		add_hook_to_module(model.lineara		,    AlignDevicesHook(execution_device=1       )       )

		self.assertEqual(model.lineara.weight.device		,    torch.device(0       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device(0       )       )
		self.assertEqual(model.batchnorm.running_mean.device		,    torch.device(0       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device(1       )       )

		# We can still make a forward pass. The input does not need to be on any particular device
		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    model(snake_case       )
		self.assertEqual(output.device		,    torch.device(1       )       )

		# We can add a general hook to put back output on same device as input.
		add_hook_to_module(snake_case		,    AlignDevicesHook(io_same_device=snake_case       )       )
		lowercase       =    torch.randn(2		,    3       ).to(0       )
		lowercase       =    model(snake_case       )
		self.assertEqual(output.device		,    torch.device(0       )       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ModelForTest()

		# Everything is on CPU
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )

		# This will move each submodule on different devices
		lowercase       =    {'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

		add_hook_to_module(model.lineara		,    AlignDevicesHook(**snake_case       )       )
		add_hook_to_module(model.batchnorm		,    AlignDevicesHook(**snake_case       )       )
		add_hook_to_module(model.lineara		,    AlignDevicesHook(**snake_case       )       )

		# Parameters have been offloaded, so on the meta device
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		# Buffers are not included in the offload by default, so are on the execution device
		lowercase       =    torch.device(hook_kwargs['execution_device']       )
		self.assertEqual(model.batchnorm.running_mean.device		,    snake_case       )

		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    model(snake_case       )
		self.assertEqual(output.device		,    snake_case       )

		# Removing hooks loads back the weights in the model.
		remove_hook_from_module(model.lineara       )
		remove_hook_from_module(model.batchnorm       )
		remove_hook_from_module(model.lineara       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )

		# Now test with buffers included in the offload
		lowercase       =    {
		    'execution_device': 0 if torch.cuda.is_available() else 'cpu',
		    'offload': True,
		    'offload_buffers': True,
		}

		add_hook_to_module(model.lineara		,    AlignDevicesHook(**snake_case       )       )
		add_hook_to_module(model.batchnorm		,    AlignDevicesHook(**snake_case       )       )
		add_hook_to_module(model.lineara		,    AlignDevicesHook(**snake_case       )       )

		# Parameters have been offloaded, so on the meta device, buffers included
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.batchnorm.running_mean.device		,    torch.device('meta'       )       )

		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    model(snake_case       )
		self.assertEqual(output.device		,    snake_case       )

		# Removing hooks loads back the weights in the model.
		remove_hook_from_module(model.lineara       )
		remove_hook_from_module(model.batchnorm       )
		remove_hook_from_module(model.lineara       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ModelForTest()

		# Everything is on CPU
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )

		# This will move each submodule on different devices
		lowercase       =    0 if torch.cuda.is_available() else 'cpu'
		attach_align_device_hook(snake_case		,    execution_device=snake_case		,    offload=snake_case       )

		# Parameters have been offloaded, so on the meta device
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		# Buffers are not included in the offload by default, so are on the execution device
		lowercase       =    torch.device(snake_case       )
		self.assertEqual(model.batchnorm.running_mean.device		,    snake_case       )

		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    model(snake_case       )
		self.assertEqual(output.device		,    snake_case       )

		# Removing hooks loads back the weights in the model.
		remove_hook_from_submodules(snake_case       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )

		# Now test with buffers included in the offload
		attach_align_device_hook(snake_case		,    execution_device=snake_case		,    offload=snake_case		,    offload_buffers=snake_case       )

		# Parameters have been offloaded, so on the meta device, buffers included
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.batchnorm.running_mean.device		,    torch.device('meta'       )       )

		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    model(snake_case       )
		self.assertEqual(output.device		,    snake_case       )

		# Removing hooks loads back the weights in the model.
		remove_hook_from_submodules(snake_case       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ModelForTest()

		# Everything is on CPU
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )

		# This will move each submodule on different devices
		lowercase       =    0 if torch.cuda.is_available() else 'cpu'
		attach_align_device_hook(
		    snake_case		,    execution_device=snake_case		,    offload=snake_case		,    weights_map=model.state_dict()       )

		# Parameters have been offloaded, so on the meta device
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		# Buffers are not included in the offload by default, so are on the execution device
		lowercase       =    torch.device(snake_case       )
		self.assertEqual(model.batchnorm.running_mean.device		,    snake_case       )

		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    model(snake_case       )
		self.assertEqual(output.device		,    snake_case       )

		# Removing hooks loads back the weights in the model.
		remove_hook_from_submodules(snake_case       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )

		# Now test with buffers included in the offload
		attach_align_device_hook(
		    snake_case		,    execution_device=snake_case		,    offload=snake_case		,    weights_map=model.state_dict()		,    offload_buffers=snake_case		,    )

		# Parameters have been offloaded, so on the meta device, buffers included
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('meta'       )       )
		self.assertEqual(model.batchnorm.running_mean.device		,    torch.device('meta'       )       )

		lowercase       =    torch.randn(2		,    3       )
		lowercase       =    model(snake_case       )
		self.assertEqual(output.device		,    snake_case       )

		# Removing hooks loads back the weights in the model.
		remove_hook_from_submodules(snake_case       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.batchnorm.weight.device		,    torch.device('cpu'       )       )
		self.assertEqual(model.lineara.weight.device		,    torch.device('cpu'       )       )



import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
	from transformers.trainer import DEFAULT_CALLBACKS

	from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel







class        A_   (			__lowerCamelCase   ):




	'''simple docstring'''


	def __init__(       self       ):
		lowercase       =    []





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_init_end'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_train_begin'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_train_end'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_epoch_begin'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_epoch_end'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_step_begin'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_step_end'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_evaluate'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_predict'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_save'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_log'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_prediction_step'       )







@require_torch
class        A_   (			unittest.TestCase   ):




	'''simple docstring'''


	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    tempfile.mkdtemp()





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		shutil.rmtree(self.output_dir       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case=0		,    snake_case=0		,    snake_case=64		,    snake_case=64		,    snake_case=None		,    snake_case=False		,    **snake_case       ):
		# disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
		# its set to False since the tests later on depend on its value.
		lowercase       =    RegressionDataset(length=snake_case       )
		lowercase       =    RegressionDataset(length=snake_case       )
		lowercase       =    RegressionModelConfig(a=snake_case		,    b=snake_case       )
		lowercase       =    RegressionPreTrainedModel(snake_case       )

		lowercase       =    TrainingArguments(self.output_dir		,    disable_tqdm=snake_case		,    report_to=[]		,    **snake_case       )
		return Trainer(
		    snake_case		,    snake_case		,    train_dataset=snake_case		,    eval_dataset=snake_case		,    callbacks=snake_case		,    )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case       ):
		self.assertEqual(len(snake_case       )		,    len(snake_case       )       )

		# Order doesn't matter
		lowercase       =    sorted(snake_case		,    key=lambda snake_case       : cb.__name__ if isinstance(snake_case		,    snake_case       ) else cb.__class__.__name__       )
		lowercase       =    sorted(snake_case		,    key=lambda snake_case       : cb.__name__ if isinstance(snake_case		,    snake_case       ) else cb.__class__.__name__       )

		for cba, cba in zip(snake_case		,    snake_case       ):
			if isinstance(snake_case		,    snake_case       ) and isinstance(snake_case		,    snake_case       ):
				self.assertEqual(snake_case		,    snake_case       )
			elif isinstance(snake_case		,    snake_case       ) and not isinstance(snake_case		,    snake_case       ):
				self.assertEqual(snake_case		,    cba.__class__       )
			elif not isinstance(snake_case		,    snake_case       ) and isinstance(snake_case		,    snake_case       ):
				self.assertEqual(cba.__class__		,    snake_case       )
			else:
				self.assertEqual(snake_case		,    snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		lowercase       =    ['on_init_end', 'on_train_begin']
		lowercase       =    0
		lowercase       =    len(trainer.get_eval_dataloader()       )
		lowercase       =    ['on_prediction_step'] * len(trainer.get_eval_dataloader()       ) + ['on_log', 'on_evaluate']
		for _ in range(trainer.state.num_train_epochs       ):
			expected_events.append('on_epoch_begin'       )
			for _ in range(snake_case       ):
				step += 1
				expected_events += ["on_step_begin", "on_step_end"]
				if step % trainer.args.logging_steps == 0:
					expected_events.append('on_log'       )
				if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
					expected_events += evaluation_events.copy()
				if step % trainer.args.save_steps == 0:
					expected_events.append('on_save'       )
			expected_events.append('on_epoch_end'       )
			if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
				expected_events += evaluation_events.copy()
		expected_events += ["on_log", "on_train_end"]
		return expected_events





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    self.get_trainer()
		lowercase       =    DEFAULT_CALLBACKS.copy() + [ProgressCallback]
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		# Callbacks passed at init are added to the default callbacks
		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]       )
		expected_callbacks.append(snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		# TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
		lowercase       =    self.get_trainer(disable_tqdm=snake_case       )
		lowercase       =    DEFAULT_CALLBACKS.copy() + [PrinterCallback]
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    DEFAULT_CALLBACKS.copy() + [ProgressCallback]
		lowercase       =    self.get_trainer()

		# We can add, pop, or remove by class name
		trainer.remove_callback(snake_case       )
		expected_callbacks.remove(snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		lowercase       =    self.get_trainer()
		lowercase       =    trainer.pop_callback(snake_case       )
		self.assertEqual(cb.__class__		,    snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		trainer.add_callback(snake_case       )
		expected_callbacks.insert(0		,    snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		# We can also add, pop, or remove by instance
		lowercase       =    self.get_trainer()
		lowercase       =    trainer.callback_handler.callbacks[0]
		trainer.remove_callback(snake_case       )
		expected_callbacks.remove(snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		lowercase       =    self.get_trainer()
		lowercase       =    trainer.callback_handler.callbacks[0]
		lowercase       =    trainer.pop_callback(snake_case       )
		self.assertEqual(snake_case		,    snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		trainer.add_callback(snake_case       )
		expected_callbacks.insert(0		,    snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		import warnings

		# XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
		warnings.simplefilter(action='ignore'		,    category=snake_case       )

		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		# Independent log/save/eval
		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]		,    logging_steps=5       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]		,    save_steps=5       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]		,    eval_steps=5		,    evaluation_strategy='steps'       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]		,    evaluation_strategy='epoch'       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		# A bit of everything
		lowercase       =    self.get_trainer(
		    callbacks=[MyTestTrainerCallback]		,    logging_steps=3		,    save_steps=10		,    eval_steps=5		,    evaluation_strategy='steps'		,    )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		# warning should be emitted for duplicated callbacks
		with patch('transformers.trainer_callback.logger.warning'       ) as warn_mock:
			lowercase       =    self.get_trainer(
			    callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]		,    )
			assert str(snake_case       ) in warn_mock.call_args[0][0]

