





import random
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
		import torch

if is_vision_available():
		import PIL

		from transformers import FlavaImageProcessor
		from transformers.image_utils import PILImageResampling
		from transformers.models.flava.image_processing_flava import (
		    FLAVA_CODEBOOK_MEAN,
		    FLAVA_CODEBOOK_STD,
		    FLAVA_IMAGE_MEAN,
		    FLAVA_IMAGE_STD,
		)
else:
		SCREAMING_SNAKE_CASE_      :			Union[str, Any]		  =      None

class 		snake_case_      (       unittest.TestCase	):







	'''simple docstring'''




	def __init__(	self       :    Optional[Any]				,			__lowerCamelCase       :    Optional[Any]				,			__lowerCamelCase       :    Tuple=7				,			__lowerCamelCase       :    Optional[int]=3				,			__lowerCamelCase       :    Tuple=30				,			__lowerCamelCase       :    Optional[int]=400				,			__lowerCamelCase       :    Any=True				,			__lowerCamelCase       :    Tuple=None				,			__lowerCamelCase       :    Optional[Any]=True				,			__lowerCamelCase       :    Union[str, Any]=None				,			__lowerCamelCase       :    Dict=None				,			__lowerCamelCase       :    Tuple=True				,			__lowerCamelCase       :    Optional[int]=1 / 255				,			__lowerCamelCase       :    Optional[Any]=True				,			__lowerCamelCase       :    Any=FLAVA_IMAGE_MEAN				,			__lowerCamelCase       :    Optional[Any]=FLAVA_IMAGE_STD				,			__lowerCamelCase       :    Dict=14				,			__lowerCamelCase       :    List[str]=75				,			__lowerCamelCase       :    Optional[Any]=None				,			__lowerCamelCase       :    List[Any]=16				,			__lowerCamelCase       :    List[Any]=0.3				,			__lowerCamelCase       :    Optional[int]=None				,			__lowerCamelCase       :    Union[str, Any]=True				,			__lowerCamelCase       :    Any=None				,			__lowerCamelCase       :    List[str]=None				,			__lowerCamelCase       :    Optional[Any]=True				,			__lowerCamelCase       :    List[str]=None				,			__lowerCamelCase       :    Union[str, Any]=True				,			__lowerCamelCase       :    Tuple=True				,			__lowerCamelCase       :    Optional[Any]=FLAVA_CODEBOOK_MEAN				,			__lowerCamelCase       :    Tuple=FLAVA_CODEBOOK_STD				,			)					->   Dict:

						'''simple docstring'''




						__lowercase					=    size if size is not None else {'height': 224, 'width': 224}
						__lowercase					=    crop_size if crop_size is not None else {'height': 224, 'width': 224}
						__lowercase					=    codebook_size if codebook_size is not None else {'height': 112, 'width': 112}
						__lowercase					=    codebook_crop_size if codebook_crop_size is not None else {'height': 112, 'width': 112}

						__lowercase					=    parent
						__lowercase					=    batch_size
						__lowercase					=    num_channels
						__lowercase					=    do_resize
						__lowercase					=    do_rescale
						__lowercase					=    rescale_factor
						__lowercase					=    min_resolution
						__lowercase					=    max_resolution
						__lowercase					=    size
						__lowercase					=    resample if resample is not None else PILImageResampling.BICUBIC
						__lowercase					=    do_normalize
						__lowercase					=    image_mean
						__lowercase					=    image_std
						__lowercase					=    do_center_crop
						__lowercase					=    crop_size

						__lowercase					=    input_size_patches
						__lowercase					=    total_mask_patches
						__lowercase					=    mask_group_max_patches
						__lowercase					=    mask_group_min_patches
						__lowercase					=    mask_group_min_aspect_ratio
						__lowercase					=    mask_group_max_aspect_ratio

						__lowercase					=    codebook_do_resize
						__lowercase					=    codebook_size
						__lowercase					=    codebook_resample if codebook_resample is not None else PILImageResampling.LANCZOS
						__lowercase					=    codebook_do_center_crop
						__lowercase					=    codebook_crop_size
						__lowercase					=    codebook_do_map_pixels
						__lowercase					=    codebook_do_normalize
						__lowercase					=    codebook_image_mean
						__lowercase					=    codebook_image_std




	def       UpperCAmelCase (	self       :    int      )					->   List[str]:

						'''simple docstring'''




						return {
						    "image_mean": self.image_mean,
						    "image_std": self.image_std,
						    "do_normalize": self.do_normalize,
						    "do_resize": self.do_resize,
						    "size": self.size,
						    "resample": self.resample,
						    "do_rescale": self.do_rescale,
						    "rescale_factor": self.rescale_factor,
						    "do_center_crop": self.do_center_crop,
						    "crop_size": self.crop_size,
						    "input_size_patches": self.input_size_patches,
						    "total_mask_patches": self.total_mask_patches,
						    "mask_group_max_patches": self.mask_group_max_patches,
						    "mask_group_min_patches": self.mask_group_min_patches,
						    "mask_group_min_aspect_ratio": self.mask_group_min_aspect_ratio,
						    "mask_group_max_aspect_ratio": self.mask_group_min_aspect_ratio,
						    "codebook_do_resize": self.codebook_do_resize,
						    "codebook_size": self.codebook_size,
						    "codebook_resample": self.codebook_resample,
						    "codebook_do_center_crop": self.codebook_do_center_crop,
						    "codebook_crop_size": self.codebook_crop_size,
						    "codebook_do_map_pixels": self.codebook_do_map_pixels,
						    "codebook_do_normalize": self.codebook_do_normalize,
						    "codebook_image_mean": self.codebook_image_mean,
						    "codebook_image_std": self.codebook_image_std,
						}




	def       UpperCAmelCase (	self       :    int      )					->   Dict:

						'''simple docstring'''




						return (self.size["height"], self.size["width"])




	def       UpperCAmelCase (	self       :    List[Any]      )					->   Any:

						'''simple docstring'''




						return (
						    (self.input_size_patches, self.input_size_patches)
						    if not isinstance(self.input_size_patches				,			__lowerCamelCase      )
						    else self.input_size_patches
						)






	def       UpperCAmelCase (	self       :    Union[str, Any]      )					->   List[Any]:

						'''simple docstring'''




						return (self.codebook_size["height"], self.codebook_size["width"])





@require_torch
@require_vision
class 		snake_case_      (       UpperCAmelCase_    ,    unittest.TestCase	):







	'''simple docstring'''
	__UpperCamelCase        =		FlavaImageProcessor if is_vision_available() else None
	__UpperCamelCase        =		None




	def       UpperCAmelCase (	self       :    Dict      )					->   Optional[int]:

						'''simple docstring'''




						__lowercase					=    FlavaImageProcessingTester(self      )




	@property
	def       UpperCAmelCase (	self       :    List[Any]      )					->   Any:

						'''simple docstring'''




						return self.image_processor_tester.prepare_image_processor_dict()




	def       UpperCAmelCase (	self       :    Union[str, Any]      )					->   Optional[int]:

						'''simple docstring'''




						__lowercase					=    self.image_processing_class(**self.image_processor_dict      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'image_mean'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'image_std'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'do_normalize'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'do_resize'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'resample'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'crop_size'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'do_center_crop'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'do_rescale'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'rescale_factor'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'masking_generator'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'codebook_do_resize'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'codebook_size'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'codebook_resample'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'codebook_do_center_crop'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'codebook_crop_size'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'codebook_do_map_pixels'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'codebook_do_normalize'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'codebook_image_mean'      )      )
						self.assertTrue(hasattr(__lowerCamelCase				,			'codebook_image_std'      )      )




	def       UpperCAmelCase (	self       :    str      )					->   str:

						'''simple docstring'''




						__lowercase					=    self.image_processing_class.from_dict(self.image_processor_dict      )
						self.assertEqual(image_processor.size				,			{'height': 224, 'width': 224}      )
						self.assertEqual(image_processor.crop_size				,			{'height': 224, 'width': 224}      )
						self.assertEqual(image_processor.codebook_size				,			{'height': 112, 'width': 112}      )
						self.assertEqual(image_processor.codebook_crop_size				,			{'height': 112, 'width': 112}      )

						__lowercase					=    self.image_processing_class.from_dict(
						    self.image_processor_dict				,			size=42				,			crop_size=84				,			codebook_size=33				,			codebook_crop_size=66      )
						self.assertEqual(image_processor.size				,			{'height': 42, 'width': 42}      )
						self.assertEqual(image_processor.crop_size				,			{'height': 84, 'width': 84}      )
						self.assertEqual(image_processor.codebook_size				,			{'height': 33, 'width': 33}      )
						self.assertEqual(image_processor.codebook_crop_size				,			{'height': 66, 'width': 66}      )




	def       UpperCAmelCase (	self       :    Any      )					->   str:

						'''simple docstring'''




						pass




	def       UpperCAmelCase (	self       :    Tuple      )					->   Any:

						'''simple docstring'''




						__lowercase					=    self.image_processing_class(**self.image_processor_dict      )
						# create random PIL images
						__lowercase					=    prepare_image_inputs(self.image_processor_tester				,			equal_resolution=__lowerCamelCase      )
						for image in image_inputs:
											self.assertIsInstance(__lowerCamelCase				,			PIL.Image.Image      )

						# Test not batched input
						__lowercase					=    image_processing(image_inputs[0]				,			return_tensors='pt'      )

						# Test no bool masked pos
						self.assertFalse('bool_masked_pos' in encoded_images      )

						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_image_size()

						self.assertEqual(
						    encoded_images.pixel_values.shape				,			(1, self.image_processor_tester.num_channels, expected_height, expected_width)				,			)

						# Test batched
						__lowercase					=    image_processing(__lowerCamelCase				,			return_tensors='pt'      )
						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_image_size()

						# Test no bool masked pos
						self.assertFalse('bool_masked_pos' in encoded_images      )

						self.assertEqual(
						    encoded_images.pixel_values.shape				,			(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        expected_height,
						        expected_width,
						    )				,			)




	def       UpperCAmelCase (	self       :    List[Any]				,			__lowerCamelCase       :    Tuple				,			__lowerCamelCase       :    Optional[int]      )					->   Dict:

						'''simple docstring'''




						__lowercase					=    self.image_processing_class(**self.image_processor_dict      )
						# create random tensors
						__lowercase					=    prepare_image_inputs(self.image_processor_tester				,			equal_resolution=__lowerCamelCase				,			**__lowerCamelCase      )
						for image in image_inputs:
											self.assertIsInstance(__lowerCamelCase				,			__lowerCamelCase      )

						# Test not batched input
						__lowercase					=    image_processing(image_inputs[0]				,			return_tensors='pt'      )

						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_image_size()
						self.assertEqual(
						    encoded_images.pixel_values.shape				,			(1, self.image_processor_tester.num_channels, expected_height, expected_width)				,			)

						__lowercase					=    image_processing(__lowerCamelCase				,			return_image_mask=__lowerCamelCase				,			return_tensors='pt'      )

						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_image_size()
						self.assertEqual(
						    encoded_images.pixel_values.shape				,			(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        expected_height,
						        expected_width,
						    )				,			)

						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_mask_size()
						self.assertEqual(
						    encoded_images.bool_masked_pos.shape				,			(
						        self.image_processor_tester.batch_size,
						        expected_height,
						        expected_width,
						    )				,			)

						# Test batched
						__lowercase					=    image_processing(__lowerCamelCase				,			return_tensors='pt'      ).pixel_values

						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_image_size()
						self.assertEqual(
						    encoded_images.shape				,			(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        expected_height,
						        expected_width,
						    )				,			)

						# Test masking
						__lowercase					=    image_processing(__lowerCamelCase				,			return_image_mask=__lowerCamelCase				,			return_tensors='pt'      )

						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_image_size()
						self.assertEqual(
						    encoded_images.pixel_values.shape				,			(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        expected_height,
						        expected_width,
						    )				,			)

						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_mask_size()
						self.assertEqual(
						    encoded_images.bool_masked_pos.shape				,			(
						        self.image_processor_tester.batch_size,
						        expected_height,
						        expected_width,
						    )				,			)




	def       UpperCAmelCase (	self       :    Any      )					->   List[str]:

						'''simple docstring'''




						self._test_call_framework(np.ndarray				,			prepare_kwargs={'numpify': True}      )




	def       UpperCAmelCase (	self       :    Optional[int]      )					->   List[str]:

						'''simple docstring'''




						self._test_call_framework(torch.Tensor				,			prepare_kwargs={'torchify': True}      )




	def       UpperCAmelCase (	self       :    Any      )					->   List[Any]:

						'''simple docstring'''




						random.seed(1_234      )
						__lowercase					=    self.image_processing_class(**self.image_processor_dict      )
						__lowercase					=    prepare_image_inputs(self.image_processor_tester				,			equal_resolution=__lowerCamelCase				,			torchify=__lowerCamelCase      )

						# Test not batched input
						__lowercase					=    image_processing(image_inputs[0]				,			return_image_mask=__lowerCamelCase				,			return_tensors='pt'      )
						self.assertEqual(encoded_images.bool_masked_pos.sum().item()				,			75      )






	def       UpperCAmelCase (	self       :    str      )					->   Tuple:

						'''simple docstring'''




						__lowercase					=    self.image_processing_class(**self.image_processor_dict      )
						# create random PIL images
						__lowercase					=    prepare_image_inputs(self.image_processor_tester				,			equal_resolution=__lowerCamelCase      )
						for image in image_inputs:
											self.assertIsInstance(__lowerCamelCase				,			PIL.Image.Image      )

						# Test not batched input
						__lowercase					=    image_processing(image_inputs[0]				,			return_codebook_pixels=__lowerCamelCase				,			return_tensors='pt'      )
						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_codebook_image_size()
						self.assertEqual(
						    encoded_images.codebook_pixel_values.shape				,			(1, self.image_processor_tester.num_channels, expected_height, expected_width)				,			)

						# Test batched
						__lowercase					=    image_processing(__lowerCamelCase				,			return_codebook_pixels=__lowerCamelCase				,			return_tensors='pt'      )
						__lowercase	,			__lowercase					=    self.image_processor_tester.get_expected_codebook_image_size()
						self.assertEqual(
						    encoded_images.codebook_pixel_values.shape				,			(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        expected_height,
						        expected_width,
						    )				,			)











import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
		from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


SCREAMING_SNAKE_CASE_      :			Union[str, Any]		  =      get_tests_dir('''fixtures/test_sentencepiece.model''')

@require_torch
@require_torchaudio
@require_sentencepiece
class 		snake_case_      (       unittest.TestCase	):







	'''simple docstring'''




	def       UpperCAmelCase (	self       :    List[Any]      )					->   str:

			'''simple docstring'''




			__lowercase					=    tempfile.mkdtemp()

			__lowercase					=    ['<s>', '<pad>', '</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est']
			__lowercase					=    dict(zip(__lowerCamelCase				,			range(len(__lowerCamelCase      )      )      )      )
			__lowercase					=    Path(self.tmpdirname      )
			save_json(__lowerCamelCase				,			save_dir / VOCAB_FILES_NAMES['vocab_file']      )
			if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
					copyfile(__lowerCamelCase				,			save_dir / VOCAB_FILES_NAMES['spm_file']      )

			__lowercase					=    SpeechaTextTokenizer.from_pretrained(self.tmpdirname      )
			tokenizer.save_pretrained(self.tmpdirname      )

			__lowercase					=    {
			    'feature_size': 24,
			    'num_mel_bins': 24,
			    'padding_value': 0.0,
			    'sampling_rate': 16_000,
			    'return_attention_mask': False,
			    'do_normalize': True,
			}
			save_json(__lowerCamelCase				,			save_dir / FEATURE_EXTRACTOR_NAME      )




	def       UpperCAmelCase (	self       :    Dict				,			**__lowerCamelCase       :    Dict      )					->   Union[str, Any]:

			'''simple docstring'''




			return SpeechaTextTokenizer.from_pretrained(self.tmpdirname				,			**__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    str				,			**__lowerCamelCase       :    List[str]      )					->   Tuple:

			'''simple docstring'''




			return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname				,			**__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    Optional[int]      )					->   str:

			'''simple docstring'''




			shutil.rmtree(self.tmpdirname      )




	def       UpperCAmelCase (	self       :    str      )					->   Any:

			'''simple docstring'''




			__lowercase					=    self.get_tokenizer()
			__lowercase					=    self.get_feature_extractor()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			processor.save_pretrained(self.tmpdirname      )
			__lowercase					=    SpeechaTextProcessor.from_pretrained(self.tmpdirname      )

			self.assertEqual(processor.tokenizer.get_vocab()				,			tokenizer.get_vocab()      )
			self.assertIsInstance(processor.tokenizer				,			__lowerCamelCase      )

			self.assertEqual(processor.feature_extractor.to_json_string()				,			feature_extractor.to_json_string()      )
			self.assertIsInstance(processor.feature_extractor				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    List[str]      )					->   Optional[Any]:

			'''simple docstring'''




			__lowercase					=    SpeechaTextProcessor(
			    tokenizer=self.get_tokenizer()				,			feature_extractor=self.get_feature_extractor()      )
			processor.save_pretrained(self.tmpdirname      )

			__lowercase					=    self.get_tokenizer(bos_token='(BOS)'				,			eos_token='(EOS)'      )
			__lowercase					=    self.get_feature_extractor(do_normalize=__lowerCamelCase				,			padding_value=1.0      )

			__lowercase					=    SpeechaTextProcessor.from_pretrained(
			    self.tmpdirname				,			bos_token='(BOS)'				,			eos_token='(EOS)'				,			do_normalize=__lowerCamelCase				,			padding_value=1.0      )

			self.assertEqual(processor.tokenizer.get_vocab()				,			tokenizer_add_kwargs.get_vocab()      )
			self.assertIsInstance(processor.tokenizer				,			__lowerCamelCase      )

			self.assertEqual(processor.feature_extractor.to_json_string()				,			feature_extractor_add_kwargs.to_json_string()      )
			self.assertIsInstance(processor.feature_extractor				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    Dict      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			__lowercase					=    floats_list((3, 1_000)      )

			__lowercase					=    feature_extractor(__lowerCamelCase				,			return_tensors='np'      )
			__lowercase					=    processor(__lowerCamelCase				,			return_tensors='np'      )

			for key in input_feat_extract.keys():
					self.assertAlmostEqual(input_feat_extract[key].sum()				,			input_processor[key].sum()				,			delta=1E-2      )




	def       UpperCAmelCase (	self       :    Union[str, Any]      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			__lowercase					=    'This is a test string'

			__lowercase					=    processor(text=__lowerCamelCase      )

			__lowercase					=    tokenizer(__lowerCamelCase      )

			for key in encoded_tok.keys():
					self.assertListEqual(encoded_tok[key]				,			encoded_processor[key]      )




	def       UpperCAmelCase (	self       :    str      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			__lowercase					=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

			__lowercase					=    processor.batch_decode(__lowerCamelCase      )
			__lowercase					=    tokenizer.batch_decode(__lowerCamelCase      )

			self.assertListEqual(__lowerCamelCase				,			__lowerCamelCase      )






	def       UpperCAmelCase (	self       :    Tuple      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			self.assertListEqual(
			    processor.model_input_names				,			feature_extractor.model_input_names				,			msg='`processor` and `feature_extractor` model input names do not match'				,			)




