




import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
     from PIL import Image

     from transformers import OwlViTImageProcessor, OwlViTProcessor




@require_vision
class       __lowerCAmelCase				(							unittest.TestCase):






       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         lowercase			:str							=  tempfile.mkdtemp()

         # fmt: off
         lowercase			:Any							=  ["", "l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
         # fmt: on
         lowercase			:Union[str, Any]							=  dict(zip(_lowerCAmelCase				,    range(len(_lowerCAmelCase ) ) ) )
         lowercase			:List[Any]							=  ["#version: 0.2", "l o", "lo w</w>", "e r</w>", ""]
         lowercase			:List[Any]							=  {"unk_token": "<unk>"}

         lowercase			:Union[str, Any]							=  os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES["vocab_file"] )
         lowercase			:str							=  os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES["merges_file"] )
         with open(self.vocab_file				,    "w"				,    encoding="utf-8" ) as fp:
           fp.write(json.dumps(_lowerCAmelCase ) + "\n" )
         with open(self.merges_file				,    "w"				,    encoding="utf-8" ) as fp:
           fp.write("\n".join(_lowerCAmelCase ) )

         lowercase			:Dict							=  {
             "do_resize": True,
             "size": 20,
             "do_center_crop": True,
             "crop_size": 18,
             "do_normalize": True,
             "image_mean": [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],
             "image_std": [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],
         }
         lowercase			:int							=  os.path.join(self.tmpdirname				,    _lowerCAmelCase )
         with open(self.image_processor_file				,    "w"				,    encoding="utf-8" ) as fp:
           json.dump(_lowerCAmelCase				,    _lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    List[Any]				,    **_lowerCAmelCase:    int ):
         return CLIPTokenizer.from_pretrained(self.tmpdirname				,    pad_token="!"				,    **_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    List[Any]				,    **_lowerCAmelCase:    Optional[int] ):
         return CLIPTokenizerFast.from_pretrained(self.tmpdirname				,    pad_token="!"				,    **_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    int				,    **_lowerCAmelCase:    Union[str, Any] ):
         return OwlViTImageProcessor.from_pretrained(self.tmpdirname				,    **_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    List[Any] ):
         shutil.rmtree(self.tmpdirname )






       def     SCREAMING_SNAKE_CASE			(      self:    Any ):
         lowercase			:Optional[int]							=  [np.random.randint(2_55				,    size=(3, 30, 4_00)				,    dtype=np.uinta )]

         lowercase			:Tuple							=  [Image.fromarray(np.moveaxis(_lowerCAmelCase				,    0				,    -1 ) ) for x in image_inputs]

         return image_inputs






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any] ):
         lowercase			:int							=  self.get_tokenizer()
         lowercase			:Optional[int]							=  self.get_rust_tokenizer()
         lowercase			:int							=  self.get_image_processor()

         lowercase			:str							=  OwlViTProcessor(tokenizer=_lowerCAmelCase				,    image_processor=_lowerCAmelCase )
         processor_slow.save_pretrained(self.tmpdirname )
         lowercase			:int							=  OwlViTProcessor.from_pretrained(self.tmpdirname				,    use_fast=_lowerCAmelCase )

         lowercase			:Union[str, Any]							=  OwlViTProcessor(tokenizer=_lowerCAmelCase				,    image_processor=_lowerCAmelCase )
         processor_fast.save_pretrained(self.tmpdirname )
         lowercase			:Tuple							=  OwlViTProcessor.from_pretrained(self.tmpdirname )

         self.assertEqual(processor_slow.tokenizer.get_vocab()				,    tokenizer_slow.get_vocab() )
         self.assertEqual(processor_fast.tokenizer.get_vocab()				,    tokenizer_fast.get_vocab() )
         self.assertEqual(tokenizer_slow.get_vocab()				,    tokenizer_fast.get_vocab() )
         self.assertIsInstance(processor_slow.tokenizer				,    _lowerCAmelCase )
         self.assertIsInstance(processor_fast.tokenizer				,    _lowerCAmelCase )

         self.assertEqual(processor_slow.image_processor.to_json_string()				,    image_processor.to_json_string() )
         self.assertEqual(processor_fast.image_processor.to_json_string()				,    image_processor.to_json_string() )
         self.assertIsInstance(processor_slow.image_processor				,    _lowerCAmelCase )
         self.assertIsInstance(processor_fast.image_processor				,    _lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         lowercase			:Dict							=  OwlViTProcessor(tokenizer=self.get_tokenizer()				,    image_processor=self.get_image_processor() )
         processor.save_pretrained(self.tmpdirname )

         lowercase			:int							=  self.get_tokenizer(bos_token="(BOS)"				,    eos_token="(EOS)" )
         lowercase			:List[Any]							=  self.get_image_processor(do_normalize=_lowerCAmelCase )

         lowercase			:Optional[Any]							=  OwlViTProcessor.from_pretrained(
             self.tmpdirname				,    bos_token="(BOS)"				,    eos_token="(EOS)"				,    do_normalize=_lowerCAmelCase )

         self.assertEqual(processor.tokenizer.get_vocab()				,    tokenizer_add_kwargs.get_vocab() )
         self.assertIsInstance(processor.tokenizer				,    _lowerCAmelCase )

         self.assertEqual(processor.image_processor.to_json_string()				,    image_processor_add_kwargs.to_json_string() )
         self.assertIsInstance(processor.image_processor				,    _lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Tuple ):
         lowercase			:List[Any]							=  self.get_image_processor()
         lowercase			:Union[str, Any]							=  self.get_tokenizer()

         lowercase			:Optional[Any]							=  OwlViTProcessor(tokenizer=_lowerCAmelCase				,    image_processor=_lowerCAmelCase )

         lowercase			:Dict							=  self.prepare_image_inputs()

         lowercase			:Union[str, Any]							=  image_processor(_lowerCAmelCase				,    return_tensors="np" )
         lowercase			:Optional[Any]							=  processor(images=_lowerCAmelCase				,    return_tensors="np" )

         for key in input_image_proc.keys():
           self.assertAlmostEqual(input_image_proc[key].sum()				,    input_processor[key].sum()				,    delta=1e-2 )






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any] ):
         lowercase			:Union[str, Any]							=  self.get_image_processor()
         lowercase			:List[Any]							=  self.get_tokenizer()

         lowercase			:Optional[Any]							=  OwlViTProcessor(tokenizer=_lowerCAmelCase				,    image_processor=_lowerCAmelCase )

         lowercase			:List[str]							=  "lower newer"

         lowercase			:Union[str, Any]							=  processor(text=_lowerCAmelCase				,    return_tensors="np" )

         lowercase			:str							=  tokenizer(_lowerCAmelCase				,    return_tensors="np" )

         for key in encoded_tok.keys():
           self.assertListEqual(encoded_tok[key][0].tolist()				,    encoded_processor[key][0].tolist() )






       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any] ):
         lowercase			:Optional[int]							=  self.get_image_processor()
         lowercase			:Optional[int]							=  self.get_tokenizer()

         lowercase			:Optional[int]							=  OwlViTProcessor(tokenizer=_lowerCAmelCase				,    image_processor=_lowerCAmelCase )

         lowercase			:Optional[int]							=  "lower newer"
         lowercase			:List[str]							=  self.prepare_image_inputs()

         lowercase			:Union[str, Any]							=  processor(text=_lowerCAmelCase				,    images=_lowerCAmelCase )

         self.assertListEqual(list(inputs.keys() )				,    ["input_ids", "attention_mask", "pixel_values"] )

         # test if it raises when no input is passed
         with pytest.raises(_lowerCAmelCase ):
           processor()






       def     SCREAMING_SNAKE_CASE			(      self:    int ):
         lowercase			:Optional[int]							=  "google/owlvit-base-patch32"
         lowercase			:int							=  OwlViTProcessor.from_pretrained(_lowerCAmelCase )

         lowercase			:Optional[int]							=  ["cat", "nasa badge"]
         lowercase			:Optional[int]							=  processor(text=_lowerCAmelCase )

         lowercase			:Dict							=  16
         self.assertListEqual(list(inputs.keys() )				,    ["input_ids", "attention_mask"] )
         self.assertEqual(inputs["input_ids"].shape				,    (2, seq_length) )

         # test if it raises when no input is passed
         with pytest.raises(_lowerCAmelCase ):
           processor()






       def     SCREAMING_SNAKE_CASE			(      self:    Tuple ):
         lowercase			:Union[str, Any]							=  "google/owlvit-base-patch32"
         lowercase			:Optional[int]							=  OwlViTProcessor.from_pretrained(_lowerCAmelCase )

         lowercase			:str							=  [["cat", "nasa badge"], ["person"]]
         lowercase			:List[Any]							=  processor(text=_lowerCAmelCase )

         lowercase			:Any							=  16
         lowercase			:int							=  len(_lowerCAmelCase )
         lowercase			:Any							=  max([len(_lowerCAmelCase ) for texts in input_texts] )

         self.assertListEqual(list(inputs.keys() )				,    ["input_ids", "attention_mask"] )
         self.assertEqual(inputs["input_ids"].shape				,    (batch_size * num_max_text_queries, seq_length) )

         # test if it raises when no input is passed
         with pytest.raises(_lowerCAmelCase ):
           processor()






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int] ):
         lowercase			:Optional[Any]							=  "google/owlvit-base-patch32"
         lowercase			:Any							=  OwlViTProcessor.from_pretrained(_lowerCAmelCase )

         lowercase			:str							=  ["cat", "nasa badge"]
         lowercase			:Dict							=  processor(text=_lowerCAmelCase )

         lowercase			:Optional[int]							=  16
         lowercase			:List[str]							=  inputs["input_ids"]
         lowercase			:Optional[Any]							=  [
             [4_94_06, 23_68, 4_94_07, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [4_94_06, 68_41, 1_13_01, 4_94_07, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         ]

         self.assertListEqual(list(inputs.keys() )				,    ["input_ids", "attention_mask"] )
         self.assertEqual(inputs["input_ids"].shape				,    (2, seq_length) )
         self.assertListEqual(list(input_ids[0] )				,    predicted_ids[0] )
         self.assertListEqual(list(input_ids[1] )				,    predicted_ids[1] )






       def     SCREAMING_SNAKE_CASE			(      self:    Dict ):
         lowercase			:Optional[int]							=  self.get_image_processor()
         lowercase			:Tuple							=  self.get_tokenizer()

         lowercase			:str							=  OwlViTProcessor(tokenizer=_lowerCAmelCase				,    image_processor=_lowerCAmelCase )

         lowercase			:List[Any]							=  self.prepare_image_inputs()
         lowercase			:List[str]							=  self.prepare_image_inputs()

         lowercase			:int							=  processor(images=_lowerCAmelCase				,    query_images=_lowerCAmelCase )

         self.assertListEqual(list(inputs.keys() )				,    ["query_pixel_values", "pixel_values"] )

         # test if it raises when no input is passed
         with pytest.raises(_lowerCAmelCase ):
           processor()






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any] ):
         lowercase			:Optional[int]							=  self.get_image_processor()
         lowercase			:List[str]							=  self.get_tokenizer()

         lowercase			:Optional[int]							=  OwlViTProcessor(tokenizer=_lowerCAmelCase				,    image_processor=_lowerCAmelCase )

         lowercase			:Any							=  [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

         lowercase			:str							=  processor.batch_decode(_lowerCAmelCase )
         lowercase			:List[str]							=  tokenizer.batch_decode(_lowerCAmelCase )

         self.assertListEqual(_lowerCAmelCase				,    _lowerCAmelCase )









import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast




@dataclass
class       __lowerCAmelCase				(							datasets.BuilderConfig):
       _a           =  None







class       __lowerCAmelCase				(							datasets.ArrowBasedBuilder):
       _a           =  PandasConfig






       def     SCREAMING_SNAKE_CASE			(      self:    Any ):
         return datasets.DatasetInfo(features=self.config.features )






       def     SCREAMING_SNAKE_CASE			(      self:    str				,    _lowerCAmelCase:    List[str] ):
         if not self.config.data_files:
           raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}" )
         lowercase			:int							=  dl_manager.download_and_extract(self.config.data_files )
         if isinstance(_lowerCAmelCase				,    (str, list, tuple) ):
           lowercase			:Union[str, Any]							=  data_files
           if isinstance(_lowerCAmelCase				,    _lowerCAmelCase ):
             lowercase			:str							=  [files]
           # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
           lowercase			:Optional[int]							=  [dl_manager.iter_files(_lowerCAmelCase ) for file in files]
           return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,    gen_kwargs={"files": files} )]
         lowercase			:List[Any]							=  []
         for split_name, files in data_files.items():
           if isinstance(_lowerCAmelCase				,    _lowerCAmelCase ):
             lowercase			:Optional[Any]							=  [files]
           # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
           lowercase			:Optional[Any]							=  [dl_manager.iter_files(_lowerCAmelCase ) for file in files]
           splits.append(datasets.SplitGenerator(name=_lowerCAmelCase				,    gen_kwargs={"files": files} ) )
         return splits






       def     SCREAMING_SNAKE_CASE			(      self:    int				,    _lowerCAmelCase:    pa.Table ):
         if self.config.features is not None:
           # more expensive cast to support nested features with keys in a different order
           # allows str <-> int/float or str to Audio for example
           lowercase			:Any							=  table_cast(_lowerCAmelCase				,    self.config.features.arrow_schema )
         return pa_table






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int]				,    _lowerCAmelCase:    Union[str, Any] ):
         for i, file in enumerate(itertools.chain.from_iterable(_lowerCAmelCase ) ):
           with open(_lowerCAmelCase				,    "rb" ) as f:
             lowercase			:int							=  pa.Table.from_pandas(pd.read_pickle(_lowerCAmelCase ) )
             yield i, self._cast_table(_lowerCAmelCase )



