



'''simple docstring'''





import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import is_speech_available, is_vision_available
from transformers.testing_utils import require_torch


if is_vision_available():
 from transformers import TvltImageProcessor

if is_speech_available():
 from transformers import TvltFeatureExtractor

from transformers import TvltProcessor






@require_torch
class        lowerCAmelCase_(				unittest.TestCase     ):
  '''simple docstring'''




  def 							UpperCAmelCase_	(    self      )    ->		str:
      lowerCAmelCase__ :  Tuple							      =					"ZinengTang/tvlt-base"
      lowerCAmelCase__ :  Optional[int]							      =					tempfile.mkdtemp()




  def 							UpperCAmelCase_	(    self  ,**__UpperCAmelCase      )    ->		Optional[Any]:
      return TvltImageProcessor.from_pretrained(self.checkpoint  ,**__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,**__UpperCAmelCase      )    ->		Tuple:
      return TvltFeatureExtractor.from_pretrained(self.checkpoint  ,**__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		List[Any]:
      shutil.rmtree(self.tmpdirname      )




  def 							UpperCAmelCase_	(    self      )    ->		Dict:
      lowerCAmelCase__ :  List[str]							      =					self.get_image_processor()
      lowerCAmelCase__ :  Optional[Any]							      =					self.get_feature_extractor()

      lowerCAmelCase__ :  Dict							      =					TvltProcessor(image_processor=__UpperCAmelCase  ,feature_extractor=__UpperCAmelCase      )
      processor.save_pretrained(self.tmpdirname      )
      lowerCAmelCase__ :  List[Any]							      =					TvltProcessor.from_pretrained(self.tmpdirname      )

      self.assertIsInstance(processor.feature_extractor  ,__UpperCAmelCase      )
      self.assertIsInstance(processor.image_processor  ,__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		List[str]:
      lowerCAmelCase__ :  int							      =					self.get_image_processor()
      lowerCAmelCase__ :  List[str]							      =					self.get_feature_extractor()

      lowerCAmelCase__ :  Optional[int]							      =					TvltProcessor(image_processor=__UpperCAmelCase  ,feature_extractor=__UpperCAmelCase      )

      lowerCAmelCase__ :  Optional[int]							      =					np.ones([1_2000]      )

      lowerCAmelCase__ :  Dict							      =					feature_extractor(__UpperCAmelCase  ,return_tensors="""np"""      )
      lowerCAmelCase__ :  Optional[int]							      =					processor(audio=__UpperCAmelCase  ,return_tensors="""np"""      )

      for key in audio_dict.keys():
          self.assertAlmostEqual(audio_dict[key].sum()  ,input_processor[key].sum()  ,delta=1E-2      )




  def 							UpperCAmelCase_	(    self      )    ->		Dict:
      lowerCAmelCase__ :  Any							      =					self.get_image_processor()
      lowerCAmelCase__ :  Any							      =					self.get_feature_extractor()

      lowerCAmelCase__ :  Union[str, Any]							      =					TvltProcessor(image_processor=__UpperCAmelCase  ,feature_extractor=__UpperCAmelCase      )

      lowerCAmelCase__ :  Optional[Any]							      =					np.ones([3, 224, 224]      )

      lowerCAmelCase__ :  List[Any]							      =					image_processor(__UpperCAmelCase  ,return_tensors="""np"""      )
      lowerCAmelCase__ :  List[Any]							      =					processor(images=__UpperCAmelCase  ,return_tensors="""np"""      )

      for key in image_dict.keys():
          self.assertAlmostEqual(image_dict[key].sum()  ,input_processor[key].sum()  ,delta=1E-2      )




  def 							UpperCAmelCase_	(    self      )    ->		List[Any]:
      lowerCAmelCase__ :  Optional[int]							      =					self.get_image_processor()
      lowerCAmelCase__ :  Any							      =					self.get_feature_extractor()

      lowerCAmelCase__ :  int							      =					TvltProcessor(image_processor=__UpperCAmelCase  ,feature_extractor=__UpperCAmelCase      )

      lowerCAmelCase__ :  List[Any]							      =					np.ones([1_2000]      )
      lowerCAmelCase__ :  str							      =					np.ones([3, 224, 224]      )

      lowerCAmelCase__ :  Any							      =					processor(audio=__UpperCAmelCase  ,images=__UpperCAmelCase      )

      self.assertListEqual(list(inputs.keys()      )  ,["""audio_values""", """audio_mask""", """pixel_values""", """pixel_mask"""]      )

      # test if it raises when no input is passed
      with pytest.raises(__UpperCAmelCase      ):
          processor()






  def 							UpperCAmelCase_	(    self      )    ->		Union[str, Any]:
      lowerCAmelCase__ :  Dict							      =					self.get_image_processor()
      lowerCAmelCase__ :  Tuple							      =					self.get_feature_extractor()

      lowerCAmelCase__ :  Any							      =					TvltProcessor(image_processor=__UpperCAmelCase  ,feature_extractor=__UpperCAmelCase      )

      self.assertListEqual(
          processor.model_input_names  ,image_processor.model_input_names + feature_extractor.model_input_names  ,msg="""`processor` and `image_processor`+`feature_extractor` model input names do not match"""  ,)










import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam





class 			__a  (    datasets.BeamBasedBuilder       ):







 def 					__lowercase					(		self					:   List[str]						):


      '''simple docstring'''






      return datasets.DatasetInfo(
          features=datasets.Features({"content": datasets.Value("string"						)}						)							,   supervised_keys=SCREAMING_SNAKE_CASE							,   )







 def 					__lowercase					(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   Any						):


      '''simple docstring'''






      return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,   gen_kwargs={"examples": get_test_dummy_examples()}						)]







 def 					__lowercase					(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   Tuple							,   SCREAMING_SNAKE_CASE					:   int						):


      '''simple docstring'''






      import apache_beam as beam

      return pipeline | "Load Examples" >> beam.Create(SCREAMING_SNAKE_CASE						)







class 			__a  (    datasets.BeamBasedBuilder       ):







 def 					__lowercase					(		self					:   Any						):


      '''simple docstring'''






      return datasets.DatasetInfo(
          features=datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"						)}						)}						)							,   supervised_keys=SCREAMING_SNAKE_CASE							,   )







 def 					__lowercase					(		self					:   Any							,   SCREAMING_SNAKE_CASE					:   Tuple							,   SCREAMING_SNAKE_CASE					:   List[Any]						):


      '''simple docstring'''






      return [
          datasets.SplitGenerator(name=datasets.Split.TRAIN							,   gen_kwargs={"examples": get_test_nested_examples()}						)
      ]







 def 					__lowercase					(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   Tuple							,   SCREAMING_SNAKE_CASE					:   Any						):


      '''simple docstring'''






      import apache_beam as beam

      return pipeline | "Load Examples" >> beam.Create(SCREAMING_SNAKE_CASE						)







def 		SCREAMING_SNAKE_CASE							(					) ->	List[str]:
     return [(i, {"content": content}) for i, content in enumerate(["foo", "bar", "foobar"]			)]







def 		SCREAMING_SNAKE_CASE							(					) ->	Optional[Any]:
     return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["foo", "bar", "foobar"]			)]





class 			__a  (    A__       ):







 @require_beam
 def 					__lowercase					(		self					:   Optional[int]						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  len(get_test_dummy_examples()						)
      with tempfile.TemporaryDirectory() as tmp_cache_dir:
           UpperCamelCase__			:   List[Any]       				=  DummyBeamDataset(cache_dir=SCREAMING_SNAKE_CASE							,   beam_runner="DirectRunner"						)
           builder.download_and_prepare()
           self.assertTrue(
               os.path.exists(
                   os.path.join(SCREAMING_SNAKE_CASE							,   builder.name							,   "default"							,   "0.0.0"							,   F'{builder.name}-train.arrow'						)						)						)
           self.assertDictEqual(builder.info.features							,   datasets.Features({"content": datasets.Value("string"						)}						)						)
           UpperCamelCase__			:   Optional[Any]       				=  builder.as_dataset()
           self.assertEqual(dset["train"].num_rows							,   SCREAMING_SNAKE_CASE						)
           self.assertEqual(dset["train"].info.splits["train"].num_examples							,   SCREAMING_SNAKE_CASE						)
           self.assertDictEqual(dset["train"][0]							,   get_test_dummy_examples()[0][1]						)
           self.assertDictEqual(
               dset["train"][expected_num_examples - 1]							,   get_test_dummy_examples()[expected_num_examples - 1][1]						)
           self.assertTrue(
               os.path.exists(os.path.join(SCREAMING_SNAKE_CASE							,   builder.name							,   "default"							,   "0.0.0"							,   "dataset_info.json"						)						)						)
           del dset







 @require_beam
 def 					__lowercase					(		self					:   Any						):


      '''simple docstring'''






      import apache_beam as beam

      UpperCamelCase__			:   Any       				=  beam.io.parquetio.WriteToParquet

      UpperCamelCase__			:   Tuple       				=  len(get_test_dummy_examples()						)
      with tempfile.TemporaryDirectory() as tmp_cache_dir:
           UpperCamelCase__			:   str       				=  DummyBeamDataset(cache_dir=SCREAMING_SNAKE_CASE							,   beam_runner="DirectRunner"						)
           with patch("apache_beam.io.parquetio.WriteToParquet"						) as write_parquet_mock:
                UpperCamelCase__			:   Optional[Any]       				=  partial(SCREAMING_SNAKE_CASE							,   num_shards=2						)
                builder.download_and_prepare()
           self.assertTrue(
               os.path.exists(
                   os.path.join(
                       SCREAMING_SNAKE_CASE							,   builder.name							,   "default"							,   "0.0.0"							,   F'{builder.name}-train-00000-of-00002.arrow'						)						)						)
           self.assertTrue(
               os.path.exists(
                   os.path.join(
                       SCREAMING_SNAKE_CASE							,   builder.name							,   "default"							,   "0.0.0"							,   F'{builder.name}-train-00000-of-00002.arrow'						)						)						)
           self.assertDictEqual(builder.info.features							,   datasets.Features({"content": datasets.Value("string"						)}						)						)
           UpperCamelCase__			:   List[Any]       				=  builder.as_dataset()
           self.assertEqual(dset["train"].num_rows							,   SCREAMING_SNAKE_CASE						)
           self.assertEqual(dset["train"].info.splits["train"].num_examples							,   SCREAMING_SNAKE_CASE						)
           # Order is not preserved when sharding, so we just check that all the elements are there
           self.assertListEqual(sorted(dset["train"]["content"]						)							,   sorted(["foo", "bar", "foobar"]						)						)
           self.assertTrue(
               os.path.exists(os.path.join(SCREAMING_SNAKE_CASE							,   builder.name							,   "default"							,   "0.0.0"							,   "dataset_info.json"						)						)						)
           del dset







 @require_beam
 def 					__lowercase					(		self					:   List[Any]						):


      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_cache_dir:
           UpperCamelCase__			:   int       				=  DummyBeamDataset(cache_dir=SCREAMING_SNAKE_CASE						)
           self.assertRaises(datasets.builder.MissingBeamOptions							,   builder.download_and_prepare						)







 @require_beam
 def 					__lowercase					(		self					:   List[Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   List[Any]       				=  len(get_test_nested_examples()						)
      with tempfile.TemporaryDirectory() as tmp_cache_dir:
           UpperCamelCase__			:   Any       				=  NestedBeamDataset(cache_dir=SCREAMING_SNAKE_CASE							,   beam_runner="DirectRunner"						)
           builder.download_and_prepare()
           self.assertTrue(
               os.path.exists(
                   os.path.join(SCREAMING_SNAKE_CASE							,   builder.name							,   "default"							,   "0.0.0"							,   F'{builder.name}-train.arrow'						)						)						)
           self.assertDictEqual(
               builder.info.features							,   datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"						)}						)}						)						)
           UpperCamelCase__			:   List[Any]       				=  builder.as_dataset()
           self.assertEqual(dset["train"].num_rows							,   SCREAMING_SNAKE_CASE						)
           self.assertEqual(dset["train"].info.splits["train"].num_examples							,   SCREAMING_SNAKE_CASE						)
           self.assertDictEqual(dset["train"][0]							,   get_test_nested_examples()[0][1]						)
           self.assertDictEqual(
               dset["train"][expected_num_examples - 1]							,   get_test_nested_examples()[expected_num_examples - 1][1]						)
           self.assertTrue(
               os.path.exists(os.path.join(SCREAMING_SNAKE_CASE							,   builder.name							,   "default"							,   "0.0.0"							,   "dataset_info.json"						)						)						)
           del dset