code: 
  
  
  '''simple docstring'''
  
  import json
  import os
  import tempfile
  
  import datasets
  from datasets.arrow_writer import ArrowWriter
  from datasets.features import ArrayaD
  from utils import generate_examples, get_duration
  
  
  __lowercase:		List[Any]							=     (30, 487)
  __lowercase:		Dict							=     (36, 1_024)
  __lowercase:		List[str]							=     (100, 100)
  __lowercase:		str							=     100
  
  __lowercase:		str							=     datasets.Features(
      {"text": ArrayaD(SHAPE_TEST_1, dtype="float32"), "image": ArrayaD(SHAPE_TEST_2, dtype="float32")}
  )
  
  __lowercase	,__lowercase:		str							=     os.path.split(__file__)
  __lowercase:		List[Any]							=     os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))
  
  
  
  @get_duration
  def        SCREAMING_SNAKE_CASE__(		_UpperCamelCase      :				Dict    ,		_UpperCamelCase      :				str    ,		_UpperCamelCase      :				int   )		->					List[str]:
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
  
  
  
     with ArrowWriter(features=_UpperCamelCase    ,		path=os.path.join(_UpperCamelCase    ,		"beta.arrow"   )   ) as writer:
        for key, record in dummy_data:
           UpperCamelCase__ 						=						my_features.encode_example(_UpperCamelCase   )
           writer.write(_UpperCamelCase   )
        UpperCamelCase__						,		UpperCamelCase__ 						=						writer.finalize()
  
  
  
  @get_duration
  def        SCREAMING_SNAKE_CASE__(		_UpperCamelCase      :				int    ,		_UpperCamelCase      :				Any   )		->					Dict:
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
  
  
  
     UpperCamelCase__ 						=						datasets.Dataset.from_file(
         filename=os.path.join(_UpperCamelCase    ,		"beta.arrow"   )    ,		info=datasets.DatasetInfo(features=_UpperCamelCase   )   )
     for _ in dataset:
        pass
  
  
  
  @get_duration
  def        SCREAMING_SNAKE_CASE__(		_UpperCamelCase      :				List[Any]    ,		_UpperCamelCase      :				Optional[Any]   )		->					Optional[int]:
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
  
  
  
     UpperCamelCase__ 						=						datasets.Dataset.from_file(
         filename=os.path.join(_UpperCamelCase    ,		"beta.arrow"   )    ,		info=datasets.DatasetInfo(features=_UpperCamelCase   )   )
     dataset.set_format("numpy"   )
     for _ in dataset:
        pass
  
  
  
  @get_duration
  def        SCREAMING_SNAKE_CASE__(		_UpperCamelCase      :				Optional[int]    ,		_UpperCamelCase      :				Optional[int]   )		->					List[Any]:
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
  
  
  
     UpperCamelCase__ 						=						10
     UpperCamelCase__ 						=						datasets.Dataset.from_file(
         filename=os.path.join(_UpperCamelCase    ,		"beta.arrow"   )    ,		info=datasets.DatasetInfo(features=_UpperCamelCase   )   )
     for i in range(0    ,		len(_UpperCamelCase   )    ,		_UpperCamelCase   ):
        UpperCamelCase__ 						=						dataset[i : i + batch_size]
  
  
  
  @get_duration
  def        SCREAMING_SNAKE_CASE__(		_UpperCamelCase      :				int    ,		_UpperCamelCase      :				List[Any]   )		->					Any:
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
  
  
  
     UpperCamelCase__ 						=						10
     UpperCamelCase__ 						=						datasets.Dataset.from_file(
         filename=os.path.join(_UpperCamelCase    ,		"beta.arrow"   )    ,		info=datasets.DatasetInfo(features=_UpperCamelCase   )   )
     dataset.set_format("numpy"   )
     for i in range(0    ,		len(_UpperCamelCase   )    ,		_UpperCamelCase   ):
        UpperCamelCase__ 						=						dataset[i : i + batch_size]
  
  
  
  @get_duration
  def        SCREAMING_SNAKE_CASE__(		_UpperCamelCase      :				str    ,		_UpperCamelCase      :				List[str]   )		->					List[Any]:
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
  
  
  
     UpperCamelCase__ 						=						datasets.Dataset.from_file(
         filename=os.path.join(_UpperCamelCase    ,		"beta.arrow"   )    ,		info=datasets.DatasetInfo(features=_UpperCamelCase   )   )
     for col in feats:
        UpperCamelCase__ 						=						dataset[col]
  
  
  
  @get_duration
  def        SCREAMING_SNAKE_CASE__(		_UpperCamelCase      :				List[Any]    ,		_UpperCamelCase      :				Union[str, Any]   )		->					Tuple:
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
  
  
  
     UpperCamelCase__ 						=						datasets.Dataset.from_file(
         filename=os.path.join(_UpperCamelCase    ,		"beta.arrow"   )    ,		info=datasets.DatasetInfo(features=_UpperCamelCase   )   )
     dataset.set_format("numpy"   )
     for col in feats:
        UpperCamelCase__ 						=						dataset[col]
  
  
  
  def        SCREAMING_SNAKE_CASE__(		)		->					Tuple:
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
  
  
  
     UpperCamelCase__ 						=						{}
     UpperCamelCase__ 						=						(
         read_unformated,
         read_formatted_as_numpy,
         read_batch_unformated,
         read_batch_formatted_as_numpy,
         read_col_unformated,
         read_col_formatted_as_numpy,
     )
     with tempfile.TemporaryDirectory() as tmp_dir:
        UpperCamelCase__ 						=						datasets.Features({"image": ArrayaD(_UpperCamelCase    ,		dtype="float32"   )}   )
        UpperCamelCase__ 						=						generate_examples(features=_UpperCamelCase    ,		num_examples=_UpperCamelCase   )
        UpperCamelCase__ 						=						write(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase   )
        for read_func in read_functions:
           UpperCamelCase__ 						=						read_func(_UpperCamelCase    ,		_UpperCamelCase   )
  
     with tempfile.TemporaryDirectory() as tmp_dir:
        # don't use fixed length for fair comparison
        # feats = datasets.Features(
        #     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
        # )
        UpperCamelCase__ 						=						datasets.Features({"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"   )   )   )}   )
        UpperCamelCase__ 						=						generate_examples(
            features=_UpperCamelCase    ,		num_examples=_UpperCamelCase    ,		seq_shapes={"image": SPEED_TEST_SHAPE}   )
        UpperCamelCase__ 						=						write(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase   )
        for read_func in read_functions:
           UpperCamelCase__ 						=						read_func(_UpperCamelCase    ,		_UpperCamelCase   )
  
     with tempfile.TemporaryDirectory() as tmp_dir:
        # don't use fixed length for fair comparison
        # feats = datasets.Features(
        #     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
        # )
        UpperCamelCase__ 						=						datasets.Features({"image": datasets.Sequence(datasets.Value("float32"   )   )}   )
        UpperCamelCase__ 						=						generate_examples(
            features=_UpperCamelCase    ,		num_examples=_UpperCamelCase    ,		seq_shapes={"image": [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}    ,		)
        UpperCamelCase__ 						=						write(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase   )
        for read_func in read_functions:
           UpperCamelCase__ 						=						read_func(_UpperCamelCase    ,		_UpperCamelCase   )
  
     with open(_UpperCamelCase    ,		"wb"   ) as f:
        f.write(json.dumps(_UpperCamelCase   ).encode("utf-8"   )   )
  
  
  if __name__ == "__main__":  # useful to run the profiler
         benchmark_array_xd()
style_context: 
  
  
  '''simple docstring'''
  
  import shutil
  import tempfile
  import unittest
  
  import numpy as np
  import pytest
  
  from transformers.testing_utils import require_vision
  from transformers.utils import is_vision_available
  
  
  if is_vision_available():
         from PIL import Image
  
         from transformers import AutoProcessor, BertTokenizer, CLIPImageProcessor, GitProcessor, PreTrainedTokenizerFast
  
  
  
  
  
  
  
  @require_vision
  class  UpperCAmelCase (       unittest.TestCase):
  
   def 		lowercase_						( self :	Optional[Any]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						tempfile.mkdtemp()
  
      UpperCamelCase__ 						=						CLIPImageProcessor()
      UpperCamelCase__ 						=						BertTokenizer.from_pretrained(
          "hf-internal-testing/tiny-random-BertModel",					model_input_names=["input_ids", "attention_mask"]					)
  
      UpperCamelCase__ 						=						GitProcessor(a_,					a_					)
  
      processor.save_pretrained(self.tmpdirname					)
  
   def 		lowercase_						( self :	Union[str, Any],					**a_ :	str					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      return AutoProcessor.from_pretrained(self.tmpdirname,					**a_					).tokenizer
  
   def 		lowercase_						( self :	Union[str, Any],					**a_ :	List[str]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      return AutoProcessor.from_pretrained(self.tmpdirname,					**a_					).image_processor
  
   def 		lowercase_						( self :	int					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      shutil.rmtree(self.tmpdirname					)
  
   def 		lowercase_						( self :	str					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						[np.random.randint(255,					size=(3, 30, 400),					dtype=np.uinta					)]
  
      UpperCamelCase__ 						=						[Image.fromarray(np.moveaxis(a_,					0,					-1					)					) for x in image_inputs]
  
      return image_inputs
  
   def 		lowercase_						( self :	Any					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						GitProcessor(tokenizer=self.get_tokenizer(),					image_processor=self.get_image_processor()					)
      processor.save_pretrained(self.tmpdirname					)
  
      UpperCamelCase__ 						=						self.get_tokenizer(bos_token="(BOS)",					eos_token="(EOS)"					)
      UpperCamelCase__ 						=						self.get_image_processor(do_normalize=a_,					padding_value=1.0					)
  
      UpperCamelCase__ 						=						GitProcessor.from_pretrained(
          self.tmpdirname,					bos_token="(BOS)",					eos_token="(EOS)",					do_normalize=a_,					padding_value=1.0					)
  
      self.assertEqual(processor.tokenizer.get_vocab(),					tokenizer_add_kwargs.get_vocab()					)
      self.assertIsInstance(processor.tokenizer,					a_					)
  
      self.assertEqual(processor.image_processor.to_json_string(),					image_processor_add_kwargs.to_json_string()					)
      self.assertIsInstance(processor.image_processor,					a_					)
  
   def 		lowercase_						( self :	int					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						self.get_image_processor()
      UpperCamelCase__ 						=						self.get_tokenizer()
  
      UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)
  
      UpperCamelCase__ 						=						self.prepare_image_inputs()
  
      UpperCamelCase__ 						=						image_processor(a_,					return_tensors="np"					)
      UpperCamelCase__ 						=						processor(images=a_,					return_tensors="np"					)
  
      for key in input_feat_extract.keys():
         self.assertAlmostEqual(input_feat_extract[key].sum(),					input_processor[key].sum(),					delta=1e-2					)
  
   def 		lowercase_						( self :	Any					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						self.get_image_processor()
      UpperCamelCase__ 						=						self.get_tokenizer()
  
      UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)
  
      UpperCamelCase__ 						=						"lower newer"
  
      UpperCamelCase__ 						=						processor(text=a_					)
  
      UpperCamelCase__ 						=						tokenizer(a_,					return_token_type_ids=a_					)
  
      for key in encoded_tok.keys():
         self.assertListEqual(encoded_tok[key],					encoded_processor[key]					)
  
   def 		lowercase_						( self :	Any					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						self.get_image_processor()
      UpperCamelCase__ 						=						self.get_tokenizer()
  
      UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)
  
      UpperCamelCase__ 						=						"lower newer"
      UpperCamelCase__ 						=						self.prepare_image_inputs()
  
      UpperCamelCase__ 						=						processor(text=a_,					images=a_					)
  
      self.assertListEqual(list(inputs.keys()					),					["input_ids", "attention_mask", "pixel_values"]					)
  
      # test if it raises when no input is passed
      with pytest.raises(a_					):
         processor()
  
   def 		lowercase_						( self :	Union[str, Any]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						self.get_image_processor()
      UpperCamelCase__ 						=						self.get_tokenizer()
  
      UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)
  
      UpperCamelCase__ 						=						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
  
      UpperCamelCase__ 						=						processor.batch_decode(a_					)
      UpperCamelCase__ 						=						tokenizer.batch_decode(a_					)
  
      self.assertListEqual(a_,					a_					)
  
   def 		lowercase_						( self :	Union[str, Any]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						self.get_image_processor()
      UpperCamelCase__ 						=						self.get_tokenizer()
  
      UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)
  
      UpperCamelCase__ 						=						"lower newer"
      UpperCamelCase__ 						=						self.prepare_image_inputs()
  
      UpperCamelCase__ 						=						processor(text=a_,					images=a_					)
  
      # For now the processor supports only ['input_ids', 'attention_mask', 'pixel_values']
      self.assertListEqual(list(inputs.keys()					),					["input_ids", "attention_mask", "pixel_values"]					)