



'''simple docstring'''





import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch





def 						__snake_case(				_lowerCAmelCase		)		-> str:  # picklable for multiprocessing
      return x.sum()





def 						__snake_case(				_lowerCAmelCase		)		-> Tuple:  # picklable for multiprocessing
      return i + 1







@dataclass
class     UpperCAmelCase_       :
    """simple docstring"""







    lowercase					=		42
    lowercase					=		42







class     UpperCAmelCase_       (       _a	):
    """simple docstring"""







    def lowerCamelCase       (    self					:     str						):
          snake_case__  :    int    						=      {}
          snake_case__  :    Tuple    						=      []
          snake_case__  :    Any    						=      1
          snake_case__  :    str    						=      [1, 2]
          snake_case__  :    List[str]    						=      {"""a""": 1, """b""": 2}
          snake_case__  :    List[str]    						=      {"""a""": [1, 2], """b""": [3, 4]}
          snake_case__  :    Dict    						=      {"""a""": {"""1""": 1}, """b""": 2}
          snake_case__  :    List[Any]    						=      {"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
          snake_case__  :    List[str]    						=      {}
          snake_case__  :    List[str]    						=      []
          snake_case__  :    str    						=      2
          snake_case__  :    Dict    						=      [2, 3]
          snake_case__  :    List[Any]    						=      {"""a""": 2, """b""": 3}
          snake_case__  :    Dict    						=      {"""a""": [2, 3], """b""": [4, 5]}
          snake_case__  :    List[str]    						=      {"""a""": {"""1""": 2}, """b""": 3}
          snake_case__  :    List[Any]    						=      {"""a""": 2, """b""": 3, """c""": 4, """d""": 5}
          self.assertEqual(map_nested(snake_case_ ,     snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_						) ,     snake_case_						)

          snake_case__  :    List[str]    						=      2
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     num_proc=snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     num_proc=snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     num_proc=snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     num_proc=snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     num_proc=snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     num_proc=snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     num_proc=snake_case_						) ,     snake_case_						)
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     num_proc=snake_case_						) ,     snake_case_						)

          snake_case__  :    Tuple    						=      {"""a""": np.eye(2						), """b""": np.zeros(3						), """c""": np.ones(2						)}
          snake_case__  :    Union[str, Any]    						=      {"""a""": 2, """b""": 0, """c""": 2}
          snake_case__  :    Optional[Any]    						=      {
              """a""": np.eye(2						).astype(snake_case_						),
              """b""": np.zeros(3						).astype(snake_case_						),
              """c""": np.ones(2						).astype(snake_case_						),
          }
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     map_numpy=snake_case_						) ,     snake_case_						)
          self.assertEqual(
              {k: v.tolist() for k, v in map_nested(snake_case_ ,     snake_case_ ,     map_numpy=snake_case_						).items()} ,     {k: v.tolist() for k, v in expected_map_nested_sna_int.items()} ,     )
          self.assertEqual(map_nested(snake_case_ ,     snake_case_ ,     map_numpy=snake_case_ ,     num_proc=snake_case_						) ,     snake_case_						)
          self.assertEqual(
              {k: v.tolist() for k, v in map_nested(snake_case_ ,     snake_case_ ,     map_numpy=snake_case_ ,     num_proc=snake_case_						).items()} ,     {k: v.tolist() for k, v in expected_map_nested_sna_int.items()} ,     )
          with self.assertRaises(snake_case_						):  # can't pickle a local lambda
                map_nested(lambda snake_case_						: x + 1 ,     snake_case_ ,     num_proc=snake_case_						)




    def lowerCamelCase       (    self					:     int						):
          snake_case__  :    Tuple    						=      {"""a""": 1, """b""": 2}
          snake_case__  :    Dict    						=      {"""a""": 3, """b""": 4}
          snake_case__  :    List[str]    						=      {"""a""": 5, """b""": 6}
          snake_case__  :    Tuple    						=      sorted([("""a""", (1, 3, 5)), ("""b""", (2, 4, 6))]						)
          self.assertEqual(sorted(zip_dict(snake_case_ ,     snake_case_ ,     snake_case_						)						) ,     snake_case_						)




    def lowerCamelCase       (    self					:     int						):







          class     UpperCAmelCase_       :
              """simple docstring"""







              lowercase					=		"bar"
          snake_case__  :    Tuple    						=      Foo()
          self.assertEqual(foo.my_attr ,     """bar"""						)
          with temporary_assignment(snake_case_ ,     """my_attr""" ,     """BAR"""						):
                self.assertEqual(foo.my_attr ,     """BAR"""						)
          self.assertEqual(foo.my_attr ,     """bar"""						)





@pytest.mark.parametrize(
    """iterable_length, num_proc, expected_num_proc"""     ,   [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]     ,   )
def 						__snake_case(				_lowerCAmelCase     ,   _lowerCAmelCase     ,   _lowerCAmelCase		)		-> Optional[int]:
      with patch("""datasets.utils.py_utils._single_map_nested"""		) as mock_single_map_nested, patch(
          """datasets.parallel.parallel.Pool"""		) as mock_multiprocessing_pool:
            snake_case__  :    Union[str, Any]    						=      {f"{i}": i for i in range(_lowerCAmelCase		)}
            snake_case__  :    int    						=      map_nested(lambda _lowerCAmelCase		: x + 10     ,   _lowerCAmelCase     ,   num_proc=_lowerCAmelCase     ,   parallel_min_length=16		)
            if expected_num_proc == 1:
                  assert mock_single_map_nested.called
                  assert not mock_multiprocessing_pool.called
            else:
                  assert not mock_single_map_nested.called
                  assert mock_multiprocessing_pool.called
                  assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc







class     UpperCAmelCase_       (       _a	):
    """simple docstring"""







    @require_tf
    def lowerCamelCase       (    self					:     Optional[int]						):
          import tensorflow as tf
          from tensorflow.keras import layers

          snake_case__  :    Tuple    						=      layers.Dense(2						)

          def gen_random_output():
                snake_case__  :    Union[str, Any]    						=      tf.random.uniform((1, 3)						)
                return model(snake_case_						).numpy()

          with temp_seed(42 ,     set_tensorflow=snake_case_						):
                snake_case__  :    List[Any]    						=      gen_random_output()
          with temp_seed(42 ,     set_tensorflow=snake_case_						):
                snake_case__  :    List[str]    						=      gen_random_output()
          snake_case__  :    Any    						=      gen_random_output()

          np.testing.assert_equal(snake_case_ ,     snake_case_						)
          self.assertGreater(np.abs(outa - outa						).sum() ,     0						)




    @require_torch
    def lowerCamelCase       (    self					:     Union[str, Any]						):
          import torch

          def gen_random_output():
                snake_case__  :    List[str]    						=      torch.nn.Linear(3 ,     2						)
                snake_case__  :    Tuple    						=      torch.rand(1 ,     3						)
                return model(snake_case_						).detach().numpy()

          with temp_seed(42 ,     set_pytorch=snake_case_						):
                snake_case__  :    List[str]    						=      gen_random_output()
          with temp_seed(42 ,     set_pytorch=snake_case_						):
                snake_case__  :    List[str]    						=      gen_random_output()
          snake_case__  :    List[str]    						=      gen_random_output()

          np.testing.assert_equal(snake_case_ ,     snake_case_						)
          self.assertGreater(np.abs(outa - outa						).sum() ,     0						)




    def lowerCamelCase       (    self					:     Optional[Any]						):
          def gen_random_output():
                return np.random.rand(1 ,     3						)

          with temp_seed(42						):
                snake_case__  :    Union[str, Any]    						=      gen_random_output()
          with temp_seed(42						):
                snake_case__  :    List[str]    						=      gen_random_output()
          snake_case__  :    Optional[int]    						=      gen_random_output()

          np.testing.assert_equal(snake_case_ ,     snake_case_						)
          self.assertGreater(np.abs(outa - outa						).sum() ,     0						)





@pytest.mark.parametrize("""input_data"""     ,   [{}]		)
def 						__snake_case(				_lowerCAmelCase		)		-> Union[str, Any]:
      snake_case__  :    Any    						=      NestedDataStructure(_lowerCAmelCase		).data
      assert output_data == input_data





@pytest.mark.parametrize(
    """data, expected_output"""     ,   [
        ({}, []),
        ([], []),
        ("""foo""", ["""foo"""]),
        (["""foo""", """bar"""], ["""foo""", """bar"""]),
        ([["""foo""", """bar"""]], ["""foo""", """bar"""]),
        ([[["""foo"""], ["""bar"""]]], ["""foo""", """bar"""]),
        ([[["""foo"""], """bar"""]], ["""foo""", """bar"""]),
        ({"""a""": 1, """b""": 2}, [1, 2]),
        ({"""a""": [1, 2], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, [4]]}, [1, 2, 3, 4]),
        ({"""a""": {"""1""": 1}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": [2]}, [1, 2]),
    ]     ,   )
def 						__snake_case(				_lowerCAmelCase     ,   _lowerCAmelCase		)		-> Union[str, Any]:
      snake_case__  :    Dict    						=      NestedDataStructure(_lowerCAmelCase		).flatten()
      assert output == expected_output





def 						__snake_case(				)		-> Any:
      snake_case__  :    Optional[int]    						=      A(x=1     ,   y="""foobar"""		)
      snake_case__  :    Any    						=      {"""x""": 1, """y""": """foobar"""}
      assert asdict(_lowerCAmelCase		) == expected_output

      snake_case__  :    str    						=      {"""a""": {"""b""": A(x=10     ,   y="""foo"""		)}, """c""": [A(x=20     ,   y="""bar"""		)]}
      snake_case__  :    Optional[int]    						=      {"""a""": {"""b""": {"""x""": 10, """y""": """foo"""}}, """c""": [{"""x""": 20, """y""": """bar"""}]}
      assert asdict(_lowerCAmelCase		) == expected_output

      with pytest.raises(_lowerCAmelCase		):
            asdict([1, A(x=10     ,   y="""foo"""		)]		)





def 						__snake_case(				_lowerCAmelCase		)		-> Optional[int]:
      return text.split()





def 						__snake_case(				_lowerCAmelCase		)		-> Optional[int]:
      yield (time.time(), content)
      time.sleep(2		)
      yield (time.time(), content)





def 						__snake_case(				)		-> Optional[Any]:
      with Pool(2		) as pool:
            snake_case__  :    Tuple    						=      list(iflatmap_unordered(_lowerCAmelCase     ,   _split_text     ,   kwargs_iterable=[{"""text""": """hello there"""}] * 10		)		)
            assert out.count("""hello"""		) == 10
            assert out.count("""there"""		) == 10
            assert len(_lowerCAmelCase		) == 20

      # check multiprocess from pathos (uses dill for pickling)
      with multiprocess.Pool(2		) as pool:
            snake_case__  :    Optional[Any]    						=      list(iflatmap_unordered(_lowerCAmelCase     ,   _split_text     ,   kwargs_iterable=[{"""text""": """hello there"""}] * 10		)		)
            assert out.count("""hello"""		) == 10
            assert out.count("""there"""		) == 10
            assert len(_lowerCAmelCase		) == 20

      # check that we get items as fast as possible
      with Pool(2		) as pool:
            snake_case__  :    str    						=      []
            for yield_time, content in iflatmap_unordered(
                _lowerCAmelCase     ,   _aseconds_generator_of_aitems_with_timing     ,   kwargs_iterable=[{"""content""": """a"""}, {"""content""": """b"""}]		):
                  assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
                  out.append(_lowerCAmelCase		)
            assert out.count("""a"""		) == 2
            assert out.count("""b"""		) == 2
            assert len(_lowerCAmelCase		) == 4





import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_vision_available


if is_vision_available():
				from PIL import Image

				from transformers import ChineseCLIPImageProcessor, ChineseCLIPProcessor



@require_vision
class 	_lowercase							(   unittest.TestCase	):





	"""simple docstring"""




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						a					     =   tempfile.mkdtemp()

						a					     =   [
						    "[UNK]",
						    "[CLS]",
						    "[SEP]",
						    "[PAD]",
						    "[MASK]",
						    "的",
						    "价",
						    "格",
						    "是",
						    "15",
						    "便",
						    "alex",
						    "##andra",
						    "，",
						    "。",
						    "-",
						    "t",
						    "shirt",
						]
						a					     =   os.path.join(self.tmpdirname		,       VOCAB_FILES_NAMES["vocab_file"]		)
						with open(self.vocab_file		,       "w"		,       encoding="utf-8"		) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)

						a					     =   {
						    "do_resize": True,
						    "size": {"height": 224, "width": 224},
						    "do_center_crop": True,
						    "crop_size": {"height": 18, "width": 18},
						    "do_normalize": True,
						    "image_mean": [0.4814_5466, 0.457_8275, 0.4082_1073],
						    "image_std": [0.2686_2954, 0.2613_0258, 0.2757_7711],
						    "do_convert_rgb": True,
						}
						a					     =   os.path.join(self.tmpdirname		,       lowerCamelCase_		)
						with open(self.image_processor_file		,       "w"		,       encoding="utf-8"		) as fp:
											json.dump(lowerCamelCase_		,       lowerCamelCase_		)




	def      UpperCamelCase_       (self		,       **lowerCamelCase_		):
						"""simple docstring"""

						return BertTokenizer.from_pretrained(self.tmpdirname		,       **lowerCamelCase_		)




	def      UpperCamelCase_       (self		,       **lowerCamelCase_		):
						"""simple docstring"""

						return BertTokenizerFast.from_pretrained(self.tmpdirname		,       **lowerCamelCase_		)




	def      UpperCamelCase_       (self		,       **lowerCamelCase_		):
						"""simple docstring"""

						return ChineseCLIPImageProcessor.from_pretrained(self.tmpdirname		,       **lowerCamelCase_		)




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						shutil.rmtree(self.tmpdirname		)




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						a					     =   [np.random.randint(255		,       size=(3, 30, 400)		,       dtype=np.uinta		)]

						a					     =   [Image.fromarray(np.moveaxis(lowerCamelCase_		,       0		,       -1		)		) for x in image_inputs]

						return image_inputs




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						a					     =   self.get_tokenizer()
						a					     =   self.get_rust_tokenizer()
						a					     =   self.get_image_processor()

						a					     =   ChineseCLIPProcessor(tokenizer=lowerCamelCase_		,       image_processor=lowerCamelCase_		)
						processor_slow.save_pretrained(self.tmpdirname		)
						a					     =   ChineseCLIPProcessor.from_pretrained(self.tmpdirname		,       use_fast=lowerCamelCase_		)

						a					     =   ChineseCLIPProcessor(tokenizer=lowerCamelCase_		,       image_processor=lowerCamelCase_		)
						processor_fast.save_pretrained(self.tmpdirname		)
						a					     =   ChineseCLIPProcessor.from_pretrained(self.tmpdirname		)

						self.assertEqual(processor_slow.tokenizer.get_vocab()		,       tokenizer_slow.get_vocab()		)
						self.assertEqual(processor_fast.tokenizer.get_vocab()		,       tokenizer_fast.get_vocab()		)
						self.assertEqual(tokenizer_slow.get_vocab()		,       tokenizer_fast.get_vocab()		)
						self.assertIsInstance(processor_slow.tokenizer		,       lowerCamelCase_		)
						self.assertIsInstance(processor_fast.tokenizer		,       lowerCamelCase_		)

						self.assertEqual(processor_slow.image_processor.to_json_string()		,       image_processor.to_json_string()		)
						self.assertEqual(processor_fast.image_processor.to_json_string()		,       image_processor.to_json_string()		)
						self.assertIsInstance(processor_slow.image_processor		,       lowerCamelCase_		)
						self.assertIsInstance(processor_fast.image_processor		,       lowerCamelCase_		)




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						a					     =   ChineseCLIPProcessor(tokenizer=self.get_tokenizer()		,       image_processor=self.get_image_processor()		)
						processor.save_pretrained(self.tmpdirname		)

						a					     =   self.get_tokenizer(cls_token="(CLS)"		,       sep_token="(SEP)"		)
						a					     =   self.get_image_processor(do_normalize=lowerCamelCase_		)

						a					     =   ChineseCLIPProcessor.from_pretrained(
						    self.tmpdirname		,       cls_token="(CLS)"		,       sep_token="(SEP)"		,       do_normalize=lowerCamelCase_		)

						self.assertEqual(processor.tokenizer.get_vocab()		,       tokenizer_add_kwargs.get_vocab()		)
						self.assertIsInstance(processor.tokenizer		,       lowerCamelCase_		)

						self.assertEqual(processor.image_processor.to_json_string()		,       image_processor_add_kwargs.to_json_string()		)
						self.assertIsInstance(processor.image_processor		,       lowerCamelCase_		)




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						a					     =   self.get_image_processor()
						a					     =   self.get_tokenizer()

						a					     =   ChineseCLIPProcessor(tokenizer=lowerCamelCase_		,       image_processor=lowerCamelCase_		)

						a					     =   self.prepare_image_inputs()

						a					     =   image_processor(lowerCamelCase_		,       return_tensors="np"		)
						a					     =   processor(images=lowerCamelCase_		,       return_tensors="np"		)

						for key in input_feat_extract.keys():
											self.assertAlmostEqual(input_feat_extract[key].sum()		,       input_processor[key].sum()		,       delta=1E-2		)




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						a					     =   self.get_image_processor()
						a					     =   self.get_tokenizer()

						a					     =   ChineseCLIPProcessor(tokenizer=lowerCamelCase_		,       image_processor=lowerCamelCase_		)

						a					     =   "Alexandra，T-shirt的价格是15便士。"

						a					     =   processor(text=lowerCamelCase_		)

						a					     =   tokenizer(lowerCamelCase_		)

						for key in encoded_tok.keys():
											self.assertListEqual(encoded_tok[key]		,       encoded_processor[key]		)




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						a					     =   self.get_image_processor()
						a					     =   self.get_tokenizer()

						a					     =   ChineseCLIPProcessor(tokenizer=lowerCamelCase_		,       image_processor=lowerCamelCase_		)

						a					     =   "Alexandra，T-shirt的价格是15便士。"
						a					     =   self.prepare_image_inputs()

						a					     =   processor(text=lowerCamelCase_		,       images=lowerCamelCase_		)

						self.assertListEqual(list(inputs.keys()		)		,       ["input_ids", "token_type_ids", "attention_mask", "pixel_values"]		)

						# test if it raises when no input is passed
						with pytest.raises(lowerCamelCase_		):
											processor()




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						a					     =   self.get_image_processor()
						a					     =   self.get_tokenizer()

						a					     =   ChineseCLIPProcessor(tokenizer=lowerCamelCase_		,       image_processor=lowerCamelCase_		)

						a					     =   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

						a					     =   processor.batch_decode(lowerCamelCase_		)
						a					     =   tokenizer.batch_decode(lowerCamelCase_		)

						self.assertListEqual(lowerCamelCase_		,       lowerCamelCase_		)




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						a					     =   self.get_image_processor()
						a					     =   self.get_tokenizer()

						a					     =   ChineseCLIPProcessor(tokenizer=lowerCamelCase_		,       image_processor=lowerCamelCase_		)

						a					     =   "Alexandra，T-shirt的价格是15便士。"
						a					     =   self.prepare_image_inputs()

						a					     =   processor(text=lowerCamelCase_		,       images=lowerCamelCase_		)

						self.assertListEqual(list(inputs.keys()		)		,       processor.model_input_names		)
