






import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
def 						_a    (      a :Features )							->			Optional[int]:
 a		  =      np.inf

 def set_batch_size(a :FeatureType ) -> None:
  nonlocal batch_size
  if isinstance(a	,							a ):
   a		  =      min(a	,							config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS )
  elif isinstance(a	,							a ):
   a		  =      min(a	,							config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS )
  elif isinstance(a	,							a ) and feature.dtype == "binary":
   a		  =      min(a	,							config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS )

 _visit(a	,							a )

 return None if batch_size is np.inf else batch_size






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      def __init__(      self      :						str ,					__UpperCAmelCase      :						NestedDataStructureLike[PathLike] ,					__UpperCAmelCase      :						Optional[NamedSplit] = None ,					__UpperCAmelCase      :						Optional[Features] = None ,					__UpperCAmelCase      :						str = None ,					__UpperCAmelCase      :						bool = False ,					__UpperCAmelCase      :						bool = False ,					__UpperCAmelCase      :						Optional[int] = None ,					**__UpperCAmelCase      :						List[Any] ,					)    ->List[Any]:




       """simple docstring"""


       super().__init__(
           __UpperCAmelCase ,					split=__UpperCAmelCase ,					features=__UpperCAmelCase ,					cache_dir=__UpperCAmelCase ,					keep_in_memory=__UpperCAmelCase ,					streaming=__UpperCAmelCase ,					num_proc=__UpperCAmelCase ,					**__UpperCAmelCase ,					)
       a		  =      path_or_paths if isinstance(__UpperCAmelCase ,					__UpperCAmelCase  ) else {self.split: path_or_paths}
       a		  =      _PACKAGED_DATASETS_MODULES['''parquet'''][1]
       a		  =      Parquet(
           cache_dir=__UpperCAmelCase ,					data_files=__UpperCAmelCase ,					features=__UpperCAmelCase ,					hash=__UpperCAmelCase ,					**__UpperCAmelCase ,					)







      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->Optional[int]:




       """simple docstring"""


       if self.streaming:
        a		  =      self.builder.as_streaming_dataset(split=self.split  )
       # Build regular (map-style) dataset
       else:
        a		  =      None
        a		  =      None
        a		  =      None
        a		  =      None

        self.builder.download_and_prepare(
            download_config=__UpperCAmelCase ,					download_mode=__UpperCAmelCase ,					verification_mode=__UpperCAmelCase ,					base_path=__UpperCAmelCase ,					num_proc=self.num_proc ,					)
        a		  =      self.builder.as_dataset(
            split=self.split ,					verification_mode=__UpperCAmelCase ,					in_memory=self.keep_in_memory  )
       return dataset






class 		lowercase_					:






      '''simple docstring'''




      def __init__(      self      :						Union[str, Any] ,					__UpperCAmelCase      :						Dataset ,					__UpperCAmelCase      :						Union[PathLike, BinaryIO] ,					__UpperCAmelCase      :						Optional[int] = None ,					**__UpperCAmelCase      :						List[str] ,					)    ->Any:




       """simple docstring"""


       a		  =      dataset
       a		  =      path_or_buf
       a		  =      batch_size or get_writer_batch_size(dataset.features  )
       a		  =      parquet_writer_kwargs







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->int:




       """simple docstring"""


       a		  =      self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

       if isinstance(self.path_or_buf ,					(str, bytes, os.PathLike)  ):
        with open(self.path_or_buf ,					'''wb+'''  ) as buffer:
         a		  =      self._write(file_obj=__UpperCAmelCase ,					batch_size=__UpperCAmelCase ,					**self.parquet_writer_kwargs  )
       else:
        a		  =      self._write(file_obj=self.path_or_buf ,					batch_size=__UpperCAmelCase ,					**self.parquet_writer_kwargs  )
       return written







      def       __lowerCAmelCase					(      self      :						List[Any] ,					__UpperCAmelCase      :						BinaryIO ,					__UpperCAmelCase      :						int ,					**__UpperCAmelCase      :						List[str]  )    ->int:




       """simple docstring"""


       a		  =      0
       a		  =      parquet_writer_kwargs.pop('''path_or_buf''' ,					__UpperCAmelCase  )
       a		  =      self.dataset.features.arrow_schema

       a		  =      pq.ParquetWriter(__UpperCAmelCase ,					schema=__UpperCAmelCase ,					**__UpperCAmelCase  )

       for offset in logging.tqdm(
           range(0 ,					len(self.dataset  ) ,					__UpperCAmelCase  ) ,					unit='''ba''' ,					disable=not logging.is_progress_bar_enabled() ,					desc='''Creating parquet from Arrow format''' ,					):
        a		  =      query_table(
            table=self.dataset._data ,					key=slice(__UpperCAmelCase ,					offset + batch_size  ) ,					indices=self.dataset._indices if self.dataset._indices is not None else None ,					)
        writer.write_table(__UpperCAmelCase  )
        written += batch.nbytes
       writer.close()
       return written









'''simple docstring'''


import unittest

from transformers.testing_utils import CaptureStdout
from transformers.tools.python_interpreter import evaluate
def 							UpperCAmelCase_  (						__lowerCamelCase					:					List[Any] ):
   return x + 2


class       a_   (				unittest.TestCase							):





     def 							lowercase__      ( self		:					Optional[Any]						):





        """simple docstring"""




        lowercase_			:Optional[int]    			= "x = 3"
        lowercase_			:Any    			= {}
        lowercase_			:Any    			= evaluate(lowercase	,      {}	,      state=lowercase						)
        assert result == 3
        self.assertDictEqual(lowercase	,      {"x": 3}						)

        lowercase_			:Dict    			= "x = y"
        lowercase_			:Dict    			= {"y": 5}
        lowercase_			:str    			= evaluate(lowercase	,      {}	,      state=lowercase						)
        # evaluate returns the value of the last assignment.
        assert result == 5
        self.assertDictEqual(lowercase	,      {"x": 5, "y": 5}						)





     def 							lowercase__      ( self		:					List[Any]						):





        """simple docstring"""




        lowercase_			:Dict    			= "y = add_two(x)"
        lowercase_			:Optional[Any]    			= {"x": 3}
        lowercase_			:int    			= evaluate(lowercase	,      {"add_two": add_two}	,      state=lowercase						)
        assert result == 5
        self.assertDictEqual(lowercase	,      {"x": 3, "y": 5}						)

        # Won't work without the tool
        with CaptureStdout() as out:
           lowercase_			:List[Any]    			= evaluate(lowercase	,      {}	,      state=lowercase						)
        assert result is None
        assert "tried to execute add_two" in out.out





     def 							lowercase__      ( self		:					int						):





        """simple docstring"""




        lowercase_			:Optional[int]    			= "x = 3"
        lowercase_			:Any    			= {}
        lowercase_			:Any    			= evaluate(lowercase	,      {}	,      state=lowercase						)
        assert result == 3
        self.assertDictEqual(lowercase	,      {"x": 3}						)





     def 							lowercase__      ( self		:					Any						):





        """simple docstring"""




        lowercase_			:Optional[Any]    			= "test_dict = {'x': x, 'y': add_two(x)}"
        lowercase_			:List[str]    			= {"x": 3}
        lowercase_			:Any    			= evaluate(lowercase	,      {"add_two": add_two}	,      state=lowercase						)
        self.assertDictEqual(lowercase	,      {"x": 3, "y": 5}						)
        self.assertDictEqual(lowercase	,      {"x": 3, "test_dict": {"x": 3, "y": 5}}						)





     def 							lowercase__      ( self		:					Optional[Any]						):





        """simple docstring"""




        lowercase_			:Optional[int]    			= "x = 3\ny = 5"
        lowercase_			:Union[str, Any]    			= {}
        lowercase_			:Optional[int]    			= evaluate(lowercase	,      {}	,      state=lowercase						)
        # evaluate returns the value of the last assignment.
        assert result == 5
        self.assertDictEqual(lowercase	,      {"x": 3, "y": 5}						)





     def 							lowercase__      ( self		:					List[Any]						):





        """simple docstring"""




        lowercase_			:List[str]    			= "text = f'This is x: {x}.'"
        lowercase_			:int    			= {"x": 3}
        lowercase_			:int    			= evaluate(lowercase	,      {}	,      state=lowercase						)
        # evaluate returns the value of the last assignment.
        assert result == "This is x: 3."
        self.assertDictEqual(lowercase	,      {"x": 3, "text": "This is x: 3."}						)





     def 							lowercase__      ( self		:					Optional[int]						):





        """simple docstring"""




        lowercase_			:List[Any]    			= "if x <= 3:\n    y = 2\nelse:\n    y = 5"
        lowercase_			:List[str]    			= {"x": 3}
        lowercase_			:Union[str, Any]    			= evaluate(lowercase	,      {}	,      state=lowercase						)
        # evaluate returns the value of the last assignment.
        assert result == 2
        self.assertDictEqual(lowercase	,      {"x": 3, "y": 2}						)

        lowercase_			:List[str]    			= {"x": 8}
        lowercase_			:Tuple    			= evaluate(lowercase	,      {}	,      state=lowercase						)
        # evaluate returns the value of the last assignment.
        assert result == 5
        self.assertDictEqual(lowercase	,      {"x": 8, "y": 5}						)





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        lowercase_			:Any    			= "test_list = [x, add_two(x)]"
        lowercase_			:Optional[int]    			= {"x": 3}
        lowercase_			:List[str]    			= evaluate(lowercase	,      {"add_two": add_two}	,      state=lowercase						)
        self.assertListEqual(lowercase	,      [3, 5]						)
        self.assertDictEqual(lowercase	,      {"x": 3, "test_list": [3, 5]}						)





     def 							lowercase__      ( self		:					List[Any]						):





        """simple docstring"""




        lowercase_			:str    			= "y = x"
        lowercase_			:Any    			= {"x": 3}
        lowercase_			:Tuple    			= evaluate(lowercase	,      {}	,      state=lowercase						)
        assert result == 3
        self.assertDictEqual(lowercase	,      {"x": 3, "y": 3}						)





     def 							lowercase__      ( self		:					Any						):





        """simple docstring"""




        lowercase_			:List[Any]    			= "test_list = [x, add_two(x)]\ntest_list[1]"
        lowercase_			:Dict    			= {"x": 3}
        lowercase_			:Any    			= evaluate(lowercase	,      {"add_two": add_two}	,      state=lowercase						)
        assert result == 5
        self.assertDictEqual(lowercase	,      {"x": 3, "test_list": [3, 5]}						)

        lowercase_			:Union[str, Any]    			= "test_dict = {'x': x, 'y': add_two(x)}\ntest_dict['y']"
        lowercase_			:List[Any]    			= {"x": 3}
        lowercase_			:List[Any]    			= evaluate(lowercase	,      {"add_two": add_two}	,      state=lowercase						)
        assert result == 5
        self.assertDictEqual(lowercase	,      {"x": 3, "test_dict": {"x": 3, "y": 5}}						)







     def 							lowercase__      ( self		:					Any						):





        """simple docstring"""




        lowercase_			:Dict    			= "x = 0\nfor i in range(3):\n    x = i"
        lowercase_			:Any    			= {}
        lowercase_			:Tuple    			= evaluate(lowercase	,      {"range": range}	,      state=lowercase						)
        assert result == 2
        self.assertDictEqual(lowercase	,      {"x": 2, "i": 2}						)


