



import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil


class       a							(			__lowerCamelCase				):



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      List[str]     		=	pa.array(TypedSequence([1, 2, 3]  )  )
						self.assertEqual(arr.type   ,pa.intaa()  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						with self.assertRaises(__lowercase  ):
										snake_case__	:      Tuple     		=	pa.array(TypedSequence([1, 2, 3]  )   ,type=pa.intaa()  )



		def 	__lowerCamelCase			(     self     :Union[str, Any]  ):
						with self.assertRaises(__lowercase  ):
										snake_case__	:      List[Any]     		=	pa.array(TypedSequence([1, 2, 3]   ,try_type=Value('''bool'''  )   ,type=Value('''int64'''  )  )  )



		def 	__lowerCamelCase			(     self     :Dict  ):
						snake_case__	:      int     		=	pa.array(TypedSequence([1, 2, 3]   ,type=Value('''int32'''  )  )  )
						self.assertEqual(arr.type   ,pa.intaa()  )



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						with self.assertRaises((TypeError, pa.lib.ArrowInvalid)  ):
										snake_case__	:      List[str]     		=	pa.array(TypedSequence(['''foo''', '''bar''']   ,type=Value('''int64'''  )  )  )



		def 	__lowerCamelCase			(     self     :Union[str, Any]  ):
						snake_case__	:      Union[str, Any]     		=	pa.array(TypedSequence([1, 2, 3]   ,try_type=Value('''int32'''  )  )  )
						self.assertEqual(arr.type   ,pa.intaa()  )



		def 	__lowerCamelCase			(     self     :Optional[int]  ):
						snake_case__	:      Union[str, Any]     		=	pa.array(TypedSequence(['''foo''', '''bar''']   ,try_type=Value('''int64'''  )  )  )
						self.assertEqual(arr.type   ,pa.string()  )



		def 	__lowerCamelCase			(     self     :Any  ):
						snake_case__	:      Tuple     		=	pa.array(TypedSequence([[[1, 2, 3]]]   ,type=ArrayaD((1, 3)   ,'''int64'''  )  )  )
						self.assertEqual(arr.type   ,ArrayaDExtensionType((1, 3)   ,'''int64'''  )  )



		def 	__lowerCamelCase			(     self     :int  ):
						with self.assertRaises((TypeError, pa.lib.ArrowInvalid)  ):
										snake_case__	:      Any     		=	pa.array(TypedSequence(['''foo''', '''bar''']   ,type=ArrayaD((1, 3)   ,'''int64'''  )  )  )



		def 	__lowerCamelCase			(     self     :Optional[Any]  ):
						snake_case__	:      int     		=	pa.array(TypedSequence([[[1, 2, 3]]]   ,try_type=ArrayaD((1, 3)   ,'''int64'''  )  )  )
						self.assertEqual(arr.type   ,ArrayaDExtensionType((1, 3)   ,'''int64'''  )  )



		def 	__lowerCamelCase			(     self     :Dict  ):
						snake_case__	:      Optional[Any]     		=	pa.array(TypedSequence(['''foo''', '''bar''']   ,try_type=ArrayaD((1, 3)   ,'''int64'''  )  )  )
						self.assertEqual(arr.type   ,pa.string()  )






		@require_pil
		def 	__lowerCamelCase			(     self     :Dict  ):
						import PIL.Image

						snake_case__	:      str     		=	PIL.Image.fromarray(np.arange(1_0   ,dtype=np.uinta  ).reshape(2   ,5  )  )
						with patch(
						    '''datasets.arrow_writer.cast_to_python_objects'''   ,side_effect=__lowercase  ) as mock_cast_to_python_objects:
										snake_case__	:      Union[str, Any]     		=	pa.array(TypedSequence([{'''path''': None, '''bytes''': b'''image_bytes'''}, pil_image]   ,type=Image()  )  )
										snake_case__		, snake_case__	:      Any     		=	mock_cast_to_python_objects.call_args_list[-1]
										self.assertIn('''optimize_list_casting'''   ,__lowercase  )
										self.assertFalse(kwargs['''optimize_list_casting''']  )



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					List[str]:


				"""simple docstring"""

				snake_case__	:      List[Any]     		=	pa.BufferReader(__lowerCAmelCase      ) if isinstance(__lowerCAmelCase  ,  pa.Buffer      ) else pa.memory_map(__lowerCAmelCase      )
				snake_case__	:      Tuple     		=	pa.ipc.open_stream(__lowerCAmelCase      )
				snake_case__	:      pa.Table     		=	f.read_all()
				assert len(pa_table.to_batches()      ) == expected_num_chunks
				assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
				del pa_table



@pytest.mark.parametrize('''writer_batch_size'''  ,  [None, 1, 10]      )
@pytest.mark.parametrize(
    '''fields'''  ,  [None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					Optional[Any]:


				"""simple docstring"""

				snake_case__	:      str     		=	pa.BufferOutputStream()
				snake_case__	:      int     		=	pa.schema(__lowerCAmelCase      ) if fields else None
				with ArrowWriter(stream=__lowerCAmelCase  ,  schema=__lowerCAmelCase  ,  writer_batch_size=__lowerCAmelCase      ) as writer:
								writer.write({'''col_1''': '''foo''', '''col_2''': 1}      )
								writer.write({'''col_1''': '''bar''', '''col_2''': 2}      )
								snake_case__		, snake_case__	:      Optional[Any]     		=	writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								snake_case__	:      str     		=	{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(__lowerCAmelCase  ,  metadata=writer._schema.metadata      )
				_check_output(output.getvalue()  ,  expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )



def 			_lowerCAmelCase       (					)   ->					int:


				"""simple docstring"""

				snake_case__	:      Optional[Any]     		=	pa.BufferOutputStream()
				snake_case__	:      Optional[Any]     		=	Features({'''labels''': ClassLabel(names=['''neg''', '''pos''']      )}      )
				with ArrowWriter(stream=__lowerCAmelCase  ,  features=__lowerCAmelCase      ) as writer:
								writer.write({'''labels''': 0}      )
								writer.write({'''labels''': 1}      )
								snake_case__		, snake_case__	:      int     		=	writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				assert writer._schema == features.arrow_schema
				assert writer._schema.metadata == features.arrow_schema.metadata
				snake_case__	:      Tuple     		=	pa.BufferReader(output.getvalue()      )
				snake_case__	:      Dict     		=	pa.ipc.open_stream(__lowerCAmelCase      )
				snake_case__	:      pa.Table     		=	f.read_all()
				snake_case__	:      str     		=	pa_table.schema
				assert pa_table.num_rows == 2
				assert schema == features.arrow_schema
				assert schema.metadata == features.arrow_schema.metadata
				assert features == Features.from_arrow_schema(__lowerCAmelCase      )



@pytest.mark.parametrize('''writer_batch_size'''  ,  [None, 1, 10]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					int:


				"""simple docstring"""

				snake_case__	:      Union[str, Any]     		=	pa.BufferOutputStream()
				with ArrowWriter(
				    stream=__lowerCAmelCase  ,  writer_batch_size=__lowerCAmelCase  ,  hash_salt='''split_name'''  ,  check_duplicates=__lowerCAmelCase  ,  ) as writer:
								with pytest.raises(__lowerCAmelCase      ):
												writer.write({'''col_1''': '''foo''', '''col_2''': 1}  ,  key=[1, 2]      )
												snake_case__		, snake_case__	:      Any     		=	writer.finalize()



@pytest.mark.parametrize('''writer_batch_size'''  ,  [None, 2, 10]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					Optional[int]:


				"""simple docstring"""

				snake_case__	:      Any     		=	pa.BufferOutputStream()
				with ArrowWriter(
				    stream=__lowerCAmelCase  ,  writer_batch_size=__lowerCAmelCase  ,  hash_salt='''split_name'''  ,  check_duplicates=__lowerCAmelCase  ,  ) as writer:
								with pytest.raises(__lowerCAmelCase      ):
												writer.write({'''col_1''': '''foo''', '''col_2''': 1}  ,  key=10      )
												writer.write({'''col_1''': '''bar''', '''col_2''': 2}  ,  key=10      )
												snake_case__		, snake_case__	:      Tuple     		=	writer.finalize()



@pytest.mark.parametrize('''writer_batch_size'''  ,  [None, 2, 10]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					str:


				"""simple docstring"""

				snake_case__	:      Dict     		=	pa.BufferOutputStream()
				with ArrowWriter(
				    stream=__lowerCAmelCase  ,  writer_batch_size=__lowerCAmelCase  ,  hash_salt='''split_name'''  ,  check_duplicates=__lowerCAmelCase  ,  ) as writer:
								writer.write({'''col_1''': '''foo''', '''col_2''': 1}  ,  key=1      )
								writer.write({'''col_1''': '''bar''', '''col_2''': 2}  ,  key=2      )
								snake_case__		, snake_case__	:      List[Any]     		=	writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				_check_output(output.getvalue()  ,  expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )



@pytest.mark.parametrize('''writer_batch_size'''  ,  [None, 1, 10]      )
@pytest.mark.parametrize(
    '''fields'''  ,  [None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					str:


				"""simple docstring"""

				snake_case__	:      Optional[Any]     		=	pa.BufferOutputStream()
				snake_case__	:      Optional[int]     		=	pa.schema(__lowerCAmelCase      ) if fields else None
				with ArrowWriter(stream=__lowerCAmelCase  ,  schema=__lowerCAmelCase  ,  writer_batch_size=__lowerCAmelCase      ) as writer:
								writer.write_batch({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}      )
								writer.write_batch({'''col_1''': [], '''col_2''': []}      )
								snake_case__		, snake_case__	:      Optional[int]     		=	writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								snake_case__	:      Optional[int]     		=	{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(__lowerCAmelCase  ,  metadata=writer._schema.metadata      )
				_check_output(output.getvalue()  ,  expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )



@pytest.mark.parametrize('''writer_batch_size'''  ,  [None, 1, 10]      )
@pytest.mark.parametrize(
    '''fields'''  ,  [None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					Tuple:


				"""simple docstring"""

				snake_case__	:      Tuple     		=	pa.BufferOutputStream()
				snake_case__	:      str     		=	pa.schema(__lowerCAmelCase      ) if fields else None
				with ArrowWriter(stream=__lowerCAmelCase  ,  schema=__lowerCAmelCase  ,  writer_batch_size=__lowerCAmelCase      ) as writer:
								writer.write_table(pa.Table.from_pydict({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}      )      )
								snake_case__		, snake_case__	:      int     		=	writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								snake_case__	:      str     		=	{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(__lowerCAmelCase  ,  metadata=writer._schema.metadata      )
				_check_output(output.getvalue()  ,  expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )



@pytest.mark.parametrize('''writer_batch_size'''  ,  [None, 1, 10]      )
@pytest.mark.parametrize(
    '''fields'''  ,  [None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					Optional[Any]:


				"""simple docstring"""

				snake_case__	:      List[str]     		=	pa.BufferOutputStream()
				snake_case__	:      Optional[int]     		=	pa.schema(__lowerCAmelCase      ) if fields else None
				with ArrowWriter(stream=__lowerCAmelCase  ,  schema=__lowerCAmelCase  ,  writer_batch_size=__lowerCAmelCase      ) as writer:
								writer.write_row(pa.Table.from_pydict({'''col_1''': ['''foo'''], '''col_2''': [1]}      )      )
								writer.write_row(pa.Table.from_pydict({'''col_1''': ['''bar'''], '''col_2''': [2]}      )      )
								snake_case__		, snake_case__	:      Union[str, Any]     		=	writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								snake_case__	:      List[Any]     		=	{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(__lowerCAmelCase  ,  metadata=writer._schema.metadata      )
				_check_output(output.getvalue()  ,  expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )



def 			_lowerCAmelCase       (					)   ->					Dict:


				"""simple docstring"""

				with tempfile.TemporaryDirectory() as tmp_dir:
								snake_case__	:      Optional[int]     		=	{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
								snake_case__	:      Dict     		=	os.path.join(__lowerCAmelCase  ,  '''test.arrow'''      )
								with ArrowWriter(path=__lowerCAmelCase  ,  schema=pa.schema(__lowerCAmelCase      )      ) as writer:
												writer.write_batch({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}      )
												snake_case__		, snake_case__	:      Tuple     		=	writer.finalize()
								assert num_examples == 2
								assert num_bytes > 0
								assert writer._schema == pa.schema(__lowerCAmelCase  ,  metadata=writer._schema.metadata      )
								_check_output(__lowerCAmelCase  ,  1      )



def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					List[str]:


				"""simple docstring"""

				if pa.types.is_list(__lowerCAmelCase      ):
								return get_base_dtype(arr_type.value_type      )
				else:
								return arr_type



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					int:


				"""simple docstring"""

				if isinstance(lst[0]  ,  __lowerCAmelCase      ):
								change_first_primitive_element_in_list(lst[0]  ,  __lowerCAmelCase      )
				else:
								snake_case__	:      List[Any]     		=	value



@pytest.mark.parametrize('''optimized_int_type, expected_dtype'''  ,  [(None, pa.intaa()), (Value('''int32'''      ), pa.intaa())]      )
@pytest.mark.parametrize('''sequence'''  ,  [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase      )   ->					str:


				"""simple docstring"""

				snake_case__	:      Dict     		=	pa.array(TypedSequence(__lowerCAmelCase  ,  optimized_int_type=__lowerCAmelCase      )      )
				assert get_base_dtype(arr.type      ) == expected_dtype



@pytest.mark.parametrize(
    '''col, expected_dtype'''  ,  [
        ('''attention_mask''', pa.inta()),
        ('''special_tokens_mask''', pa.inta()),
        ('''token_type_ids''', pa.inta()),
        ('''input_ids''', pa.intaa()),
        ('''other''', pa.intaa()),
    ]  ,  )
@pytest.mark.parametrize('''sequence'''  ,  [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase      )   ->					int:


				"""simple docstring"""

				snake_case__	:      List[str]     		=	pa.array(OptimizedTypedSequence(__lowerCAmelCase  ,  col=__lowerCAmelCase      )      )
				assert get_base_dtype(arr.type      ) == expected_dtype

				# not in range
				if col != "other":
								# avoids errors due to in-place modifications
								snake_case__	:      Tuple     		=	copy.deepcopy(__lowerCAmelCase      )
								snake_case__	:      int     		=	np.iinfo(expected_dtype.to_pandas_dtype()      ).max + 1
								change_first_primitive_element_in_list(__lowerCAmelCase  ,  __lowerCAmelCase      )
								snake_case__	:      Optional[Any]     		=	pa.array(OptimizedTypedSequence(__lowerCAmelCase  ,  col=__lowerCAmelCase      )      )
								assert get_base_dtype(arr.type      ) == pa.intaa()



@pytest.mark.parametrize('''raise_exception'''  ,  [False, True]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					List[str]:


				"""simple docstring"""

				snake_case__	:      Optional[Any]     		=	str(tmp_path / '''dataset-train.arrow'''      )
				try:
								with ArrowWriter(path=__lowerCAmelCase      ) as writer:
												if raise_exception:
																raise pa.lib.ArrowInvalid()
												else:
																writer.stream.close()
				except pa.lib.ArrowInvalid:
								pass
				finally:
								assert writer.stream.closed



def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					int:


				"""simple docstring"""

				snake_case__	:      int     		=	'''mock://dataset-train.arrow'''
				with ArrowWriter(path=__lowerCAmelCase  ,  storage_options=mockfs.storage_options      ) as writer:
								assert isinstance(writer._fs  ,  type(__lowerCAmelCase      )      )
								assert writer._fs.storage_options == mockfs.storage_options
								writer.write({'''col_1''': '''foo''', '''col_2''': 1}      )
								writer.write({'''col_1''': '''bar''', '''col_2''': 2}      )
								snake_case__		, snake_case__	:      str     		=	writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				assert mockfs.exists(__lowerCAmelCase      )



def 			_lowerCAmelCase       (					)   ->					List[str]:


				"""simple docstring"""

				snake_case__	:      Optional[Any]     		=	pa.BufferOutputStream()
				with ParquetWriter(stream=__lowerCAmelCase      ) as writer:
								writer.write({'''col_1''': '''foo''', '''col_2''': 1}      )
								writer.write({'''col_1''': '''bar''', '''col_2''': 2}      )
								snake_case__		, snake_case__	:      Any     		=	writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				snake_case__	:      str     		=	pa.BufferReader(output.getvalue()      )
				snake_case__	:      pa.Table     		=	pq.read_table(__lowerCAmelCase      )
				assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}



@require_pil
@pytest.mark.parametrize('''embed_local_files'''  ,  [False, True]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					Any:


				"""simple docstring"""

				import PIL.Image

				snake_case__	:      Dict     		=	str(tmp_path / '''test_image_rgb.jpg'''      )
				PIL.Image.fromarray(np.zeros((5, 5)  ,  dtype=np.uinta      )      ).save(__lowerCAmelCase  ,  format='''png'''      )
				snake_case__	:      Tuple     		=	pa.BufferOutputStream()
				with ParquetWriter(
				    stream=__lowerCAmelCase  ,  features=Features({'''image''': Image()}      )  ,  embed_local_files=__lowerCAmelCase      ) as writer:
								writer.write({'''image''': image_path}      )
								writer.finalize()
				snake_case__	:      List[Any]     		=	pa.BufferReader(output.getvalue()      )
				snake_case__	:      pa.Table     		=	pq.read_table(__lowerCAmelCase      )
				snake_case__	:      Optional[Any]     		=	pa_table.to_pydict()
				if embed_local_files:
								assert isinstance(out['''image'''][0]['''path''']  ,  __lowerCAmelCase      )
								with open(__lowerCAmelCase  ,  '''rb'''      ) as f:
												assert out["image"][0]["bytes"] == f.read()
				else:
								assert out["image"][0]["path"] == image_path
								assert out["image"][0]["bytes"] is None





def 			_lowerCAmelCase       (					)   ->					Dict:


				"""simple docstring"""

				snake_case__	:      Optional[Any]     		=	pa.schema([pa.field('''col_1'''  ,  pa.string()  ,  nullable=__lowerCAmelCase      )]      )
				snake_case__	:      Tuple     		=	pa.BufferOutputStream()
				with ArrowWriter(stream=__lowerCAmelCase      ) as writer:
								writer._build_writer(inferred_schema=__lowerCAmelCase      )
								assert writer._schema == pa.schema([pa.field('''col_1'''  ,  pa.string()      )]      )






import random

from .binary_exp_mod import bin_exp_mod



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase=1000      )   ->					str:


				"""simple docstring"""

				if n < 2:
								return False

				if n % 2 == 0:
								return n == 2

				# this means n is odd
				snake_case__	:      Tuple     		=	n - 1
				snake_case__	:      Tuple     		=	0
				while d % 2 == 0:
								d /= 2
								exp += 1

				# n - 1=d*(2**exp)
				snake_case__	:      List[str]     		=	0
				while count < prec:
								snake_case__	:      List[str]     		=	random.randint(2  ,  n - 1      )
								snake_case__	:      Optional[Any]     		=	bin_exp_mod(__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase      )
								if b != 1:
												snake_case__	:      List[Any]     		=	True
												for _ in range(__lowerCAmelCase      ):
																if b == n - 1:
																				snake_case__	:      List[str]     		=	False
																				break
																snake_case__	:      str     		=	b * b
																b %= n
												if flag:
																return False
												count += 1
				return True


if __name__ == "__main__":
				A__								=						abs(int(input('''Enter bound : ''').strip()))
				print('''Here\'s the list of primes:''')
				print(''', '''.join(str(i) for i in range(n + 1) if is_prime_big(i)))

