
'''simple docstring'''


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
	import torch

if is_vision_available():
	from PIL import Image

	from transformers import TvltImageProcessor
def 					__magic_name__(       UpperCAmelCase__ :       str							,				UpperCAmelCase__ :       Optional[Any]=10							,				UpperCAmelCase__ :       List[str]=10							,				UpperCAmelCase__ :       Optional[Any]=False							,				UpperCAmelCase__ :       List[Any]=False	):
						UpperCamelCase_     :	List[str]					     =     []
						for i in range(image_processor_tester.num_frames	):
												video.append(np.random.randint(255							,				size=(image_processor_tester.num_channels, width, height)							,				dtype=np.uinta	)	)

						if not numpify and not torchify:
												# PIL expects the channel dimension as last dimension
												UpperCamelCase_     :	List[Any]					     =     [Image.fromarray(np.moveaxis(UpperCAmelCase__							,				0							,				-1	)	) for frame in video]

						if torchify:
												UpperCamelCase_     :	int					     =     [torch.from_numpy(UpperCAmelCase__	) for frame in video]

						return video
def 					__magic_name__(       UpperCAmelCase__ :       List[str]							,				UpperCAmelCase__ :       List[str]=False							,				UpperCAmelCase__ :       Any=False							,				UpperCAmelCase__ :       List[str]=False	):
						assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

						UpperCamelCase_     :	Union[str, Any]					     =     []
						for i in range(image_processor_tester.batch_size	):
												if equal_resolution:
																		UpperCamelCase_     :	str					     =     image_processor_tester.max_resolution
												else:
																		UpperCamelCase_     ,UpperCamelCase_     :	Optional[Any]					     =     np.random.choice(
																		    np.arange(image_processor_tester.min_resolution							,				image_processor_tester.max_resolution	)							,				2	)
																		UpperCamelCase_     :	Optional[int]					     =     prepare_video(
																		    image_processor_tester=UpperCAmelCase__							,				width=UpperCAmelCase__							,				height=UpperCAmelCase__							,				numpify=UpperCAmelCase__							,				torchify=UpperCAmelCase__							,				)
												video_inputs.append(UpperCAmelCase__	)

						return video_inputs




class     __snake_case			(    unittest.TestCase	):




		def __init__(  self					:Dict			,    _SCREAMING_SNAKE_CASE					:str			,    _SCREAMING_SNAKE_CASE					:str=7			,    _SCREAMING_SNAKE_CASE					:str=3			,    _SCREAMING_SNAKE_CASE					:Any=4			,    _SCREAMING_SNAKE_CASE					:int=1_8			,    _SCREAMING_SNAKE_CASE					:Union[str, Any]=3_0			,    _SCREAMING_SNAKE_CASE					:Tuple=4_0_0			,    _SCREAMING_SNAKE_CASE					:Tuple=True			,    _SCREAMING_SNAKE_CASE					:Dict=None			,    _SCREAMING_SNAKE_CASE					:Dict=True			,    _SCREAMING_SNAKE_CASE					:List[Any]=[0.5, 0.5, 0.5]			,    _SCREAMING_SNAKE_CASE					:Any=[0.5, 0.5, 0.5]			,    _SCREAMING_SNAKE_CASE					:Tuple=True			,    _SCREAMING_SNAKE_CASE					:Tuple=None			,    )					->			Any:






								'''simple docstring'''






								UpperCamelCase_     :	Optional[int]					     =     size if size is not None else {"""shortest_edge""": 1_8}
								UpperCamelCase_     :	List[Any]					     =     crop_size if crop_size is not None else {"""height""": 1_8, """width""": 1_8}

								UpperCamelCase_     :	List[str]					     =     parent
								UpperCamelCase_     :	Union[str, Any]					     =     batch_size
								UpperCamelCase_     :	List[Any]					     =     num_channels
								UpperCamelCase_     :	int					     =     num_frames
								UpperCamelCase_     :	List[Any]					     =     image_size
								UpperCamelCase_     :	Optional[int]					     =     min_resolution
								UpperCamelCase_     :	Dict					     =     max_resolution
								UpperCamelCase_     :	Dict					     =     do_resize
								UpperCamelCase_     :	Any					     =     size
								UpperCamelCase_     :	Tuple					     =     do_normalize
								UpperCamelCase_     :	Dict					     =     image_mean
								UpperCamelCase_     :	Tuple					     =     image_std
								UpperCamelCase_     :	List[Any]					     =     do_center_crop
								UpperCamelCase_     :	Optional[int]					     =     crop_size





		def 	_snake_case	(  self					:Optional[Any])					->			List[Any]:






								'''simple docstring'''






								return {
								    "image_mean": self.image_mean,
								    "image_std": self.image_std,
								    "do_normalize": self.do_normalize,
								    "do_resize": self.do_resize,
								    "size": self.size,
								    "do_center_crop": self.do_center_crop,
								    "crop_size": self.crop_size,
								}







@require_torch
@require_vision
class     __snake_case			(    _a					,			unittest.TestCase	):
		a_          =					TvltImageProcessor if is_vision_available() else None




		def 	_snake_case	(  self					:int)					->			Union[str, Any]:






								'''simple docstring'''






								UpperCamelCase_     :	List[str]					     =     TvltImageProcessorTester(self)




		@property
		def 	_snake_case	(  self					:int)					->			Any:






								'''simple docstring'''






								return self.image_processor_tester.prepare_image_processor_dict()




		def 	_snake_case	(  self					:Dict)					->			List[Any]:






								'''simple docstring'''






								UpperCamelCase_     :	Optional[int]					     =     self.image_processing_class(**self.image_processor_dict)
								self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,    """image_mean"""))
								self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,    """image_std"""))
								self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,    """do_normalize"""))
								self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,    """do_resize"""))
								self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,    """do_center_crop"""))
								self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,    """size"""))




		def 	_snake_case	(  self					:List[Any])					->			Tuple:






								'''simple docstring'''






								UpperCamelCase_     :	Tuple					     =     self.image_processing_class(**self.image_processor_dict)
								# create random PIL videos
								UpperCamelCase_     :	Tuple					     =     prepare_video_inputs(self.image_processor_tester			,    equal_resolution=_SCREAMING_SNAKE_CASE)
								for video in video_inputs:
														self.assertIsInstance(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)
														self.assertIsInstance(video[0]			,    Image.Image)

								# Test not batched input
								UpperCamelCase_     :	Tuple					     =     image_processor(video_inputs[0]			,    return_tensors="""pt""").pixel_values
								self.assertEqual(
								    encoded_videos.shape			,    (
								        1,
								        self.image_processor_tester.num_frames,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )			,    )

								# Test batched
								UpperCamelCase_     :	int					     =     image_processor(_SCREAMING_SNAKE_CASE			,    return_tensors="""pt""").pixel_values
								self.assertEqual(
								    encoded_videos.shape			,    (
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_frames,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )			,    )




		def 	_snake_case	(  self					:Tuple)					->			Optional[Any]:






								'''simple docstring'''






								UpperCamelCase_     :	List[str]					     =     self.image_processing_class(**self.image_processor_dict)
								# create random numpy tensors
								UpperCamelCase_     :	int					     =     prepare_video_inputs(self.image_processor_tester			,    equal_resolution=_SCREAMING_SNAKE_CASE			,    numpify=_SCREAMING_SNAKE_CASE)
								for video in video_inputs:
														self.assertIsInstance(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)
														self.assertIsInstance(video[0]			,    np.ndarray)

								# Test not batched input
								UpperCamelCase_     :	List[Any]					     =     image_processor(video_inputs[0]			,    return_tensors="""pt""").pixel_values
								self.assertEqual(
								    encoded_videos.shape			,    (
								        1,
								        self.image_processor_tester.num_frames,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )			,    )

								# Test batched
								UpperCamelCase_     :	Dict					     =     image_processor(_SCREAMING_SNAKE_CASE			,    return_tensors="""pt""").pixel_values
								self.assertEqual(
								    encoded_videos.shape			,    (
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_frames,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )			,    )





		def 	_snake_case	(  self					:List[Any])					->			List[str]:






								'''simple docstring'''






								UpperCamelCase_     :	Dict					     =     self.image_processing_class(**self.image_processor_dict)
								# create random PyTorch tensors
								UpperCamelCase_     :	Union[str, Any]					     =     prepare_video_inputs(self.image_processor_tester			,    equal_resolution=_SCREAMING_SNAKE_CASE			,    torchify=_SCREAMING_SNAKE_CASE)
								for video in video_inputs:
														self.assertIsInstance(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)
														self.assertIsInstance(video[0]			,    torch.Tensor)

								# Test not batched input
								UpperCamelCase_     :	int					     =     image_processor(video_inputs[0]			,    return_tensors="""pt""").pixel_values
								self.assertEqual(
								    encoded_videos.shape			,    (
								        1,
								        self.image_processor_tester.num_frames,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )			,    )

								# Test batched
								UpperCamelCase_     :	str					     =     image_processor(_SCREAMING_SNAKE_CASE			,    return_tensors="""pt""").pixel_values
								self.assertEqual(
								    encoded_videos.shape			,    (
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_frames,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )			,    )






'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_xglm import XGLMTokenizer
else:
	__SCREAMING_SNAKE_CASE						=      None


__SCREAMING_SNAKE_CASE						=      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE						=      {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__SCREAMING_SNAKE_CASE						=      {
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json",
    },
}

__SCREAMING_SNAKE_CASE						=      {
    "facebook/xglm-564M": 2048,
}




class     __snake_case			(    _a	):
		a_          =					VOCAB_FILES_NAMES
		a_          =					PRETRAINED_VOCAB_FILES_MAP
		a_          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a_          =					["input_ids", "attention_mask"]
		a_          =					XGLMTokenizer




		def __init__(  self					:str			,    _SCREAMING_SNAKE_CASE					:Dict=None			,    _SCREAMING_SNAKE_CASE					:List[str]=None			,    _SCREAMING_SNAKE_CASE					:Dict="<s>"			,    _SCREAMING_SNAKE_CASE					:Dict="</s>"			,    _SCREAMING_SNAKE_CASE					:List[Any]="</s>"			,    _SCREAMING_SNAKE_CASE					:Tuple="<s>"			,    _SCREAMING_SNAKE_CASE					:int="<unk>"			,    _SCREAMING_SNAKE_CASE					:List[str]="<pad>"			,    **_SCREAMING_SNAKE_CASE					:Optional[int]			,    )					->			Optional[int]:






								'''simple docstring'''






								UpperCamelCase_     :	Tuple					     =     7
								UpperCamelCase_     :	int					     =     [F"""<madeupword{i}>""" for i in range(self.num_madeup_words)]

								UpperCamelCase_     :	Optional[Any]					     =     kwargs.get("""additional_special_tokens"""			,    [])
								kwargs["additional_special_tokens"] += [
								    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
								]

								super().__init__(
								    _SCREAMING_SNAKE_CASE			,    tokenizer_file=_SCREAMING_SNAKE_CASE			,    bos_token=_SCREAMING_SNAKE_CASE			,    eos_token=_SCREAMING_SNAKE_CASE			,    sep_token=_SCREAMING_SNAKE_CASE			,    cls_token=_SCREAMING_SNAKE_CASE			,    unk_token=_SCREAMING_SNAKE_CASE			,    pad_token=_SCREAMING_SNAKE_CASE			,    **_SCREAMING_SNAKE_CASE			,    )

								UpperCamelCase_     :	int					     =     vocab_file
								UpperCamelCase_     :	Any					     =     False if not self.vocab_file else True




		def 	_snake_case	(  self					:Dict			,    _SCREAMING_SNAKE_CASE					:List[int]			,    _SCREAMING_SNAKE_CASE					:Optional[List[int]] = None)					->			List[int]:






								'''simple docstring'''






								if token_ids_a is None:
														return [self.sep_token_id] + token_ids_a
								UpperCamelCase_     :	str					     =     [self.sep_token_id]
								return sep + token_ids_a + sep + sep + token_ids_a




		def 	_snake_case	(  self					:int			,    _SCREAMING_SNAKE_CASE					:List[int]			,    _SCREAMING_SNAKE_CASE					:Optional[List[int]] = None)					->			List[int]:






								'''simple docstring'''






								UpperCamelCase_     :	Optional[Any]					     =     [self.sep_token_id]

								if token_ids_a is None:
														return len(sep + token_ids_a) * [0]
								return len(sep + token_ids_a + sep + sep + token_ids_a) * [0]





		def 	_snake_case	(  self					:List[Any]			,    _SCREAMING_SNAKE_CASE					:str			,    _SCREAMING_SNAKE_CASE					:Optional[str] = None)					->			Tuple[str]:






								'''simple docstring'''






								if not self.can_save_slow_tokenizer:
														raise ValueError(
														    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
														    """tokenizer.""")

								if not os.path.isdir(_SCREAMING_SNAKE_CASE):
														logger.error(F"""Vocabulary path ({save_directory}) should be a directory.""")
														return
								UpperCamelCase_     :	Dict					     =     os.path.join(
								    _SCREAMING_SNAKE_CASE			,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

								if os.path.abspath(self.vocab_file) != os.path.abspath(_SCREAMING_SNAKE_CASE):
														copyfile(self.vocab_file			,    _SCREAMING_SNAKE_CASE)

								return (out_vocab_file,)




