code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  import timeit
  from typing import Callable, Optional
  
  from ..configuration_utils import PretrainedConfig
  from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
  from ..utils import is_pyanvml_available, is_torch_available, logging
  from .benchmark_utils import (
      Benchmark,
      Memory,
      MemorySummary,
      measure_peak_memory_cpu,
      start_memory_tracing,
      stop_memory_tracing,
  )
  
  
  if is_torch_available():
  			import torch
  
  			from .benchmark_args import PyTorchBenchmarkArguments
  
  
  if is_pyanvml_available():
  			import pyanvml.pyanvml as nvml
  
  
  a__				: Tuple									=    logging.get_logger(__name__)
  
  
  
  
  
  
  class 							__snake_case    (				__magic_name__       ):
  	__lowerCAmelCase						=     42
  	__lowerCAmelCase						=     42
  	__lowerCAmelCase						=     '''PyTorch'''
  
  
  
  
  
  	@property
  	def      _snake_case      (		self		)			->		Optional[int]:
  		return torch.__version__
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		float:
  		snake_case__  					=      self._prepare_inference_func(UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)
  		return self._measure_speed(_inference		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		[Memory, Optional[MemorySummary]]:
  		snake_case__  					=      self._prepare_inference_func(UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)
  		return self._measure_memory(_inference		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		float:
  		snake_case__  					=      self._prepare_train_func(UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)
  		return self._measure_speed(_train		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		[Memory, Optional[MemorySummary]]:
  		snake_case__  					=      self._prepare_train_func(UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)
  		return self._measure_memory(_train		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		Callable[[], None]:
  		snake_case__  					=      self.config_dict[model_name]
  
  		if self.args.torchscript:
  			snake_case__  					=      True
  
  		snake_case__  					=      (
  		    hasattr(UpperCamelCase_			,					'architectures'		)
  		    and isinstance(config.architectures			,					UpperCamelCase_		)
  		    and len(config.architectures		) > 0
  		)
  		if not self.args.only_pretrain_model and has_model_class_in_config:
  			try:
  				snake_case__  					=      config.architectures[0]
  				snake_case__  					=      __import__('transformers'			,					fromlist=[model_class]		)
  				snake_case__  					=      getattr(UpperCamelCase_			,					UpperCamelCase_		)
  				snake_case__  					=      model_cls(UpperCamelCase_		)
  			except ImportError:
  				raise ImportError(
  				    F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
  				    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'		)
  		else:
  			snake_case__  					=      MODEL_MAPPING[config.__class__](UpperCamelCase_		)
  
  		model.eval()
  		model.to(self.args.device		)
  
  		# encoder-decoder has vocab size saved differently
  		snake_case__  					=      config.vocab_size if hasattr(UpperCamelCase_			,					'vocab_size'		) else config.encoder.vocab_size
  		snake_case__  					=      torch.randint(UpperCamelCase_			,					(batch_size, sequence_length)			,					dtype=torch.long			,					device=self.args.device		)
  
  		if self.args.fpaa:
  			logger.info('Running training in Mixed Precision...'		)
  			if not self.args.is_gpu:
  				raise ValueError('Mixed precision is possible only for GPU.'		)
  			# amp seems to have memory leaks so that memory usage
  			# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
  			model.half()
  
  		if self.args.torchscript:
  			with torch.no_grad():
  				snake_case__  					=      torch.jit.trace(UpperCamelCase_			,					UpperCamelCase_		)
  		else:
  			snake_case__  					=      model
  
  		def encoder_decoder_forward():
  			with torch.no_grad():
  				snake_case__  					=      inference_model(UpperCamelCase_			,					decoder_input_ids=UpperCamelCase_		)
  			return outputs
  
  		def encoder_forward():
  			with torch.no_grad():
  				snake_case__  					=      inference_model(UpperCamelCase_		)
  			return outputs
  
  		snake_case__  					=      encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
  		return _forward
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		Callable[[], None]:
  		snake_case__  					=      self.config_dict[model_name]
  
  		snake_case__  					=      (
  		    hasattr(UpperCamelCase_			,					'architectures'		)
  		    and isinstance(config.architectures			,					UpperCamelCase_		)
  		    and len(config.architectures		) > 0
  		)
  		if not self.args.only_pretrain_model and has_model_class_in_config:
  			try:
  				snake_case__  					=      config.architectures[0]
  				snake_case__  					=      __import__('transformers'			,					fromlist=[model_class]		)
  				snake_case__  					=      getattr(UpperCamelCase_			,					UpperCamelCase_		)
  				snake_case__  					=      model_cls(UpperCamelCase_		)
  			except ImportError:
  				raise ImportError(
  				    F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
  				    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'		)
  		else:
  			snake_case__  					=      MODEL_WITH_LM_HEAD_MAPPING[config.__class__](UpperCamelCase_		)
  
  		if self.args.torchscript:
  			raise NotImplementedError('Training for torchscript is currently not implemented'		)
  		else:
  			snake_case__  					=      model
  
  		model.train()
  		model.to(self.args.device		)
  
  		# encoder-decoder has vocab size saved differently
  		snake_case__  					=      config.vocab_size if hasattr(UpperCamelCase_			,					'vocab_size'		) else config.encoder.vocab_size
  		snake_case__  					=      torch.randint(UpperCamelCase_			,					(batch_size, sequence_length)			,					dtype=torch.long			,					device=self.args.device		)
  
  		if self.args.fpaa:
  			logger.info('Running training in Mixed Precision...'		)
  			if not self.args.is_gpu:
  				raise ValueError('Mixed precision is possible only for GPU.'		)
  
  			# amp seems to have memory leaks so that memory usage
  			# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
  			model.half()
  
  		def compute_loss_and_backprob_encoder():
  			snake_case__  					=      train_model(UpperCamelCase_			,					labels=UpperCamelCase_		)[0]
  			loss.backward()
  			return loss
  
  		def compute_loss_and_backprob_encoder_decoder():
  			snake_case__  					=      train_model(UpperCamelCase_			,					decoder_input_ids=UpperCamelCase_			,					labels=UpperCamelCase_		)[0]
  			loss.backward()
  			return loss
  
  		snake_case__  					=      (
  		    compute_loss_and_backprob_encoder_decoder
  		    if config.is_encoder_decoder
  		    else compute_loss_and_backprob_encoder
  		)
  		return _train
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_		)			->		float:
  		try:
  			if self.args.is_tpu or self.args.torchscript:
  				# run additional 10 times to stabilize compilation for tpu and torchscript
  				logger.info('Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'		)
  				timeit.repeat(
  				    UpperCamelCase_			,					repeat=1			,					number=5			,					)
  
  			# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
  			snake_case__  					=      timeit.repeat(
  			    UpperCamelCase_			,					repeat=self.args.repeat			,					number=10			,					)
  
  			if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
  				import torch_xla.debug.metrics as met
  
  				self.print_fn(met.metrics_report()		)
  
  			return min(UpperCamelCase_		) / 10.0
  		except RuntimeError as e:
  			self.print_fn(F'''Doesn\'t fit on GPU. {e}'''		)
  			return "N/A"
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_		)			->		[Memory, MemorySummary]:
  		try:
  			if self.args.trace_memory_line_by_line:
  				snake_case__  					=      start_memory_tracing('transformers'		)
  
  			if self.args.is_tpu:
  				# tpu
  				raise NotImplementedError(
  				    'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'
  				    ' `--no-memory` or `args.memory=False`'		)
  			elif self.args.is_gpu:
  				if not is_pyanvml_available():
  					logger.warning(
  					    'py3nvml not installed, we won\'t log GPU memory usage. '
  					    'Install py3nvml (pip install py3nvml) to log information about GPU.'		)
  					snake_case__  					=      'N/A'
  				else:
  					logger.info(
  					    'Measuring total GPU usage on GPU device. Make sure to not have additional processes running'
  					    ' on the same GPU.'		)
  					# init nvml
  					nvml.nvmlInit()
  					func()
  					snake_case__  					=      nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx		)
  					snake_case__  					=      nvml.nvmlDeviceGetMemoryInfo(UpperCamelCase_		)
  					snake_case__  					=      meminfo.used
  					snake_case__  					=      Memory(UpperCamelCase_		)
  					# shutdown nvml
  					nvml.nvmlShutdown()
  			else:
  				# cpu
  				snake_case__  					=      measure_peak_memory_cpu(UpperCamelCase_		)
  				snake_case__  					=      Memory(UpperCamelCase_		) if isinstance(UpperCamelCase_			,					UpperCamelCase_		) else memory_bytes
  
  			if self.args.trace_memory_line_by_line:
  				snake_case__  					=      stop_memory_tracing(UpperCamelCase_		)
  			else:
  				snake_case__  					=      None
  
  			return memory, summary
  		except RuntimeError as e:
  			self.print_fn(F'''Doesn\'t fit on GPU. {e}'''		)
  			return "N/A", None
  
  
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  import json
  import os
  import random
  import shutil
  import tempfile
  import unittest
  
  import numpy as np
  import pytest
  
  from transformers import BertTokenizer, BertTokenizerFast
  from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
  from transformers.testing_utils import require_vision
  from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available
  
  
  if is_vision_available():
  			from PIL import Image
  
  			from transformers import FlavaImageProcessor, FlavaProcessor
  			from transformers.models.flava.image_processing_flava import (
  			    FLAVA_CODEBOOK_MEAN,
  			    FLAVA_CODEBOOK_STD,
  			    FLAVA_IMAGE_MEAN,
  			    FLAVA_IMAGE_STD,
  			)
  
  
  
  
  
  
  @require_vision
  class 							__snake_case    (				unittest.TestCase       ):
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[Any]:
  		snake_case__  					=      tempfile.mkdtemp()
  
  		# fmt: off
  		snake_case__  					=      ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing', ',', 'low', 'lowest']
  		# fmt: on
  		snake_case__  					=      os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['vocab_file']		)
  
  		with open(self.vocab_file			,					'w'			,					encoding='utf-8'		) as fp:
  			fp.write(''.join([x + '\n' for x in vocab_tokens]		)		)
  
  		snake_case__  					=      {
  		    'image_mean': FLAVA_IMAGE_MEAN,
  		    'image_std': FLAVA_IMAGE_STD,
  		    'do_normalize': True,
  		    'do_resize': True,
  		    'size': 224,
  		    'do_center_crop': True,
  		    'crop_size': 224,
  		    'input_size_patches': 14,
  		    'total_mask_patches': 75,
  		    'mask_group_max_patches': None,
  		    'mask_group_min_patches': 16,
  		    'mask_group_min_aspect_ratio': 0.3,
  		    'mask_group_max_aspect_ratio': None,
  		    'codebook_do_resize': True,
  		    'codebook_size': 112,
  		    'codebook_do_center_crop': True,
  		    'codebook_crop_size': 112,
  		    'codebook_do_map_pixels': True,
  		    'codebook_do_normalize': True,
  		    'codebook_image_mean': FLAVA_CODEBOOK_MEAN,
  		    'codebook_image_std': FLAVA_CODEBOOK_STD,
  		}
  
  		snake_case__  					=      os.path.join(self.tmpdirname			,					UpperCamelCase_		)
  		with open(self.image_processor_file			,					'w'			,					encoding='utf-8'		) as fp:
  			json.dump(UpperCamelCase_			,					UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self			,					**UpperCamelCase_		)			->		List[str]:
  		return BertTokenizer.from_pretrained(self.tmpdirname			,					**UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self			,					**UpperCamelCase_		)			->		Optional[Any]:
  		return BertTokenizerFast.from_pretrained(self.tmpdirname			,					**UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self			,					**UpperCamelCase_		)			->		Any:
  		return FlavaImageProcessor.from_pretrained(self.tmpdirname			,					**UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Union[str, Any]:
  		shutil.rmtree(self.tmpdirname		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[str]:
  		snake_case__  					=      [np.random.randint(255			,					size=(3, 30, 400)			,					dtype=np.uinta		)]
  
  		snake_case__  					=      [Image.fromarray(np.moveaxis(UpperCamelCase_			,					0			,					-1		)		) for x in image_inputs]
  
  		return image_inputs
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[int]:
  		snake_case__  					=      self.get_tokenizer()
  		snake_case__  					=      self.get_rust_tokenizer()
  		snake_case__  					=      self.get_image_processor()
  
  		snake_case__  					=      FlavaProcessor(tokenizer=UpperCamelCase_			,					image_processor=UpperCamelCase_		)
  		processor_slow.save_pretrained(self.tmpdirname		)
  		snake_case__  					=      FlavaProcessor.from_pretrained(self.tmpdirname			,					use_fast=UpperCamelCase_		)
  
  		snake_case__  					=      FlavaProcessor(tokenizer=UpperCamelCase_			,					image_processor=UpperCamelCase_		)
  		processor_fast.save_pretrained(self.tmpdirname		)
  		snake_case__  					=      FlavaProcessor.from_pretrained(self.tmpdirname		)
  
  		self.assertEqual(processor_slow.tokenizer.get_vocab()			,					tokenizer_slow.get_vocab()		)
  		self.assertEqual(processor_fast.tokenizer.get_vocab()			,					tokenizer_fast.get_vocab()		)
  		self.assertEqual(tokenizer_slow.get_vocab()			,					tokenizer_fast.get_vocab()		)
  		self.assertIsInstance(processor_slow.tokenizer			,					UpperCamelCase_		)
  		self.assertIsInstance(processor_fast.tokenizer			,					UpperCamelCase_		)
  
  		self.assertEqual(processor_slow.image_processor.to_json_string()			,					image_processor.to_json_string()		)
  		self.assertEqual(processor_fast.image_processor.to_json_string()			,					image_processor.to_json_string()		)
  		self.assertIsInstance(processor_slow.image_processor			,					UpperCamelCase_		)
  		self.assertIsInstance(processor_fast.image_processor			,					UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[str]:
  		snake_case__  					=      FlavaProcessor(tokenizer=self.get_tokenizer()			,					image_processor=self.get_image_processor()		)
  		processor.save_pretrained(self.tmpdirname		)
  
  		snake_case__  					=      self.get_tokenizer(bos_token='(BOS)'			,					eos_token='(EOS)'		)
  		snake_case__  					=      self.get_image_processor(do_normalize=UpperCamelCase_			,					padding_value=1.0		)
  
  		snake_case__  					=      FlavaProcessor.from_pretrained(
  		    self.tmpdirname			,					bos_token='(BOS)'			,					eos_token='(EOS)'			,					do_normalize=UpperCamelCase_			,					padding_value=1.0		)
  
  		self.assertEqual(processor.tokenizer.get_vocab()			,					tokenizer_add_kwargs.get_vocab()		)
  		self.assertIsInstance(processor.tokenizer			,					UpperCamelCase_		)
  
  		self.assertEqual(processor.image_processor.to_json_string()			,					image_processor_add_kwargs.to_json_string()		)
  		self.assertIsInstance(processor.image_processor			,					UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Union[str, Any]:
  		snake_case__  					=      self.get_image_processor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      FlavaProcessor(tokenizer=UpperCamelCase_			,					image_processor=UpperCamelCase_		)
  
  		snake_case__  					=      self.prepare_image_inputs()
  
  		snake_case__  					=      image_processor(UpperCamelCase_			,					return_tensors='np'		)
  		snake_case__  					=      processor(images=UpperCamelCase_			,					return_tensors='np'		)
  
  		for key in input_feat_extract.keys():
  			self.assertAlmostEqual(input_feat_extract[key].sum()			,					input_processor[key].sum()			,					delta=1E-2		)
  
  		# With rest of the args
  		random.seed(1234		)
  		snake_case__  					=      image_processor(
  		    UpperCamelCase_			,					return_image_mask=UpperCamelCase_			,					return_codebook_pixels=UpperCamelCase_			,					return_tensors='np'		)
  		random.seed(1234		)
  		snake_case__  					=      processor(
  		    images=UpperCamelCase_			,					return_image_mask=UpperCamelCase_			,					return_codebook_pixels=UpperCamelCase_			,					return_tensors='np'		)
  
  		for key in input_feat_extract.keys():
  			self.assertAlmostEqual(input_feat_extract[key].sum()			,					input_processor[key].sum()			,					delta=1E-2		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      self.get_image_processor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      FlavaProcessor(tokenizer=UpperCamelCase_			,					image_processor=UpperCamelCase_		)
  
  		snake_case__  					=      'lower newer'
  
  		snake_case__  					=      processor(text=UpperCamelCase_		)
  
  		snake_case__  					=      tokenizer(UpperCamelCase_		)
  
  		for key in encoded_tok.keys():
  			self.assertListEqual(encoded_tok[key]			,					encoded_processor[key]		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Union[str, Any]:
  		snake_case__  					=      self.get_image_processor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      FlavaProcessor(tokenizer=UpperCamelCase_			,					image_processor=UpperCamelCase_		)
  
  		snake_case__  					=      'lower newer'
  		snake_case__  					=      self.prepare_image_inputs()
  
  		snake_case__  					=      processor(text=UpperCamelCase_			,					images=UpperCamelCase_		)
  
  		self.assertListEqual(list(inputs.keys()		)			,					['input_ids', 'token_type_ids', 'attention_mask', 'pixel_values']		)
  
  		# add extra args
  		snake_case__  					=      processor(text=UpperCamelCase_			,					images=UpperCamelCase_			,					return_codebook_pixels=UpperCamelCase_			,					return_image_mask=UpperCamelCase_		)
  
  		self.assertListEqual(
  		    list(inputs.keys()		)			,					[
  		        'input_ids',
  		        'token_type_ids',
  		        'attention_mask',
  		        'pixel_values',
  		        'codebook_pixel_values',
  		        'bool_masked_pos',
  		    ]			,					)
  
  		# test if it raises when no input is passed
  		with pytest.raises(UpperCamelCase_		):
  			processor()
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[Any]:
  		snake_case__  					=      self.get_image_processor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      FlavaProcessor(tokenizer=UpperCamelCase_			,					image_processor=UpperCamelCase_		)
  
  		snake_case__  					=      [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
  
  		snake_case__  					=      processor.batch_decode(UpperCamelCase_		)
  		snake_case__  					=      tokenizer.batch_decode(UpperCamelCase_		)
  
  		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      self.get_image_processor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      FlavaProcessor(tokenizer=UpperCamelCase_			,					image_processor=UpperCamelCase_		)
  
  		snake_case__  					=      'lower newer'
  		snake_case__  					=      self.prepare_image_inputs()
  
  		snake_case__  					=      processor(text=UpperCamelCase_			,					images=UpperCamelCase_		)
  
  		self.assertListEqual(list(inputs.keys()		)			,					processor.model_input_names		)
  
  
  
  