


'''simple docstring'''



import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode






def 						lowerCamelCase							(_SCREAMING_SNAKE_CASE   :     List[str]		,   **_SCREAMING_SNAKE_CASE   :     List[Any]     ):
					return RunBeamCommand(
					    args.dataset		,   args.name		,   args.cache_dir		,   args.beam_pipeline_options		,   args.data_dir		,   args.all_configs		,   args.save_info or args.save_infos		,   args.ignore_verifications		,   args.force_redownload		,   **_SCREAMING_SNAKE_CASE		,   )





class 							__UpperCamelCase       (	lowerCAmelCase_      ):




	@staticmethod
	def    __UpperCAmelCase							(     __a							):

						'''simple docstring'''





						__a    : Any      					=						parser.add_parser('run_beam'		,       help='Run a Beam dataset processing pipeline'							)
						run_beam_parser.add_argument('dataset'		,       type=__a		,       help='Name of the dataset to download'							)
						run_beam_parser.add_argument('--name'		,       type=__a		,       default=__a		,       help='Dataset config name'							)
						run_beam_parser.add_argument(
						    '--cache_dir'		,       type=__a		,       default=__a		,       help='Cache directory where the datasets are stored'		,       )
						run_beam_parser.add_argument(
						    '--beam_pipeline_options'		,       type=__a		,       default=''		,       help='Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`'		,       )
						run_beam_parser.add_argument(
						    '--data_dir'		,       type=__a		,       default=__a		,       help='Can be used to specify a manual directory to get the files from'		,       )
						run_beam_parser.add_argument('--all_configs'		,       action='store_true'		,       help='Test all dataset configurations'							)
						run_beam_parser.add_argument('--save_info'		,       action='store_true'		,       help='Save the dataset infos file'							)
						run_beam_parser.add_argument(
						    '--ignore_verifications'		,       action='store_true'		,       help='Run the test without checksums and splits checks'							)
						run_beam_parser.add_argument('--force_redownload'		,       action='store_true'		,       help='Force dataset redownload'							)
						# aliases
						run_beam_parser.add_argument('--save_infos'		,       action='store_true'		,       help='alias for save_info'							)
						run_beam_parser.set_defaults(func=__a							)




	def __init__(     self		,       __a		,       __a		,       __a		,       __a		,       __a		,       __a		,       __a		,       __a		,       __a		,       **__a		,       ):

						'''simple docstring'''





						__a    : Optional[int]      					=						dataset
						__a    : str      					=						name
						__a    : Any      					=						cache_dir
						__a    : Tuple      					=						beam_pipeline_options
						__a    : str      					=						data_dir
						__a    : Optional[Any]      					=						all_configs
						__a    : List[Any]      					=						save_infos
						__a    : List[str]      					=						ignore_verifications
						__a    : Dict      					=						force_redownload
						__a    : List[Any]      					=						config_kwargs





	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						import apache_beam as beam

						if self._name is not None and self._all_configs:
											print('Both parameters `name` and `all_configs` can\'t be used at once.'							)
											exit(1							)
						__a    : Any      					=						self._dataset, self._name
						__a    : Union[str, Any]      					=						dataset_module_factory(__a							)
						__a    : str      					=						import_main_class(dataset_module.module_path							)
						__a    : List[DatasetBuilder]      					=						[]
						if self._beam_pipeline_options:
											__a    : Any      					=						beam.options.pipeline_options.PipelineOptions(
											    flags=[f"""--{opt.strip()}""" for opt in self._beam_pipeline_options.split(','							) if opt]							)
						else:
											__a    : Dict      					=						None
						if self._all_configs and len(builder_cls.BUILDER_CONFIGS							) > 0:
											for builder_config in builder_cls.BUILDER_CONFIGS:
																builders.append(
																    builder_cls(
																        config_name=builder_config.name		,       data_dir=self._data_dir		,       hash=dataset_module.hash		,       beam_options=__a		,       cache_dir=self._cache_dir		,       base_path=dataset_module.builder_kwargs.get('base_path'							)		,       )							)
						else:
											builders.append(
											    builder_cls(
											        config_name=__a		,       data_dir=self._data_dir		,       beam_options=__a		,       cache_dir=self._cache_dir		,       base_path=dataset_module.builder_kwargs.get('base_path'							)		,       **self._config_kwargs		,       )							)

						for builder in builders:
											builder.download_and_prepare(
											    download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
											    if not self._force_redownload
											    else DownloadMode.FORCE_REDOWNLOAD		,       download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH							)		,       verification_mode=VerificationMode.NO_CHECKS
											    if self._ignore_verifications
											    else VerificationMode.ALL_CHECKS		,       try_from_hf_gcs=__a		,       )
											if self._save_infos:
																builder._save_infos()

						print('Apache beam run successful.'							)

						# If save_infos=True, the dataset infos file is created next to the loaded module file.
						# Let's move it to the original directory of the dataset script, to allow the user to
						# upload them on S3 at the same time afterwards.
						if self._save_infos:
											__a    : Tuple      					=						os.path.join(builder_cls.get_imported_module_dir()		,       config.DATASETDICT_INFOS_FILENAME							)

											__a    : Optional[int]      					=						Path(__a							).name + '.py'

											__a    : Dict      					=						os.path.join(__a		,       __a							)
											if os.path.isfile(__a							):
																__a    : int      					=						os.path.dirname(__a							)
											elif os.path.isfile(__a							):
																__a    : List[Any]      					=						path
											else:  # in case of a remote dataset
																print(f"""Dataset Infos file saved at {dataset_infos_path}"""							)
																exit(1							)

											# Move datasetinfo back to the user
											__a    : str      					=						os.path.join(__a		,       config.DATASETDICT_INFOS_FILENAME							)
											copyfile(__a		,       __a							)
											print(f"""Dataset Infos file saved at {user_dataset_infos_path}"""							)







'''simple docstring'''



from __future__ import annotations

import inspect
import unittest

import numpy as np

from transformers import SwinConfig
from transformers.testing_utils import require_tf, require_vision, slow, to_atuple
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers.models.swin.modeling_tf_swin import (
	    TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
	    TFSwinForImageClassification,
	    TFSwinForMaskedImageModeling,
	    TFSwinModel,
	)


if is_vision_available():
	from PIL import Image

	from transformers import AutoImageProcessor





class 							__UpperCamelCase       :




	def __init__(     self		,       __a		,       __a=13		,       __a=32		,       __a=2		,       __a=3		,       __a=16		,       __a=[1, 2, 1]		,       __a=[2, 2, 4]		,       __a=2		,       __a=2.0		,       __a=True		,       __a=0.0		,       __a=0.0		,       __a=0.1		,       __a="gelu"		,       __a=False		,       __a=True		,       __a=0.02		,       __a=1E-5		,       __a=True		,       __a=None		,       __a=True		,       __a=10		,       __a=8		,       ):

						'''simple docstring'''





						__a    : Dict      					=						parent
						__a    : int      					=						batch_size
						__a    : str      					=						image_size
						__a    : List[Any]      					=						patch_size
						__a    : Any      					=						num_channels
						__a    : int      					=						embed_dim
						__a    : Optional[Any]      					=						depths
						__a    : Optional[Any]      					=						num_heads
						__a    : List[Any]      					=						window_size
						__a    : Dict      					=						mlp_ratio
						__a    : Any      					=						qkv_bias
						__a    : Any      					=						hidden_dropout_prob
						__a    : Union[str, Any]      					=						attention_probs_dropout_prob
						__a    : str      					=						drop_path_rate
						__a    : Dict      					=						hidden_act
						__a    : List[str]      					=						use_absolute_embeddings
						__a    : str      					=						patch_norm
						__a    : List[str]      					=						layer_norm_eps
						__a    : Optional[Any]      					=						initializer_range
						__a    : str      					=						is_training
						__a    : Any      					=						scope
						__a    : Union[str, Any]      					=						use_labels
						__a    : Optional[int]      					=						type_sequence_label_size
						__a    : Optional[Any]      					=						encoder_stride




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a    : str      					=						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

						__a    : List[Any]      					=						None
						if self.use_labels:
											__a    : str      					=						ids_tensor([self.batch_size]		,       self.type_sequence_label_size							)

						__a    : int      					=						self.get_config()

						return config, pixel_values, labels




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						return SwinConfig(
						    image_size=self.image_size		,       patch_size=self.patch_size		,       num_channels=self.num_channels		,       embed_dim=self.embed_dim		,       depths=self.depths		,       num_heads=self.num_heads		,       window_size=self.window_size		,       mlp_ratio=self.mlp_ratio		,       qkv_bias=self.qkv_bias		,       hidden_dropout_prob=self.hidden_dropout_prob		,       attention_probs_dropout_prob=self.attention_probs_dropout_prob		,       drop_path_rate=self.drop_path_rate		,       hidden_act=self.hidden_act		,       use_absolute_embeddings=self.use_absolute_embeddings		,       path_norm=self.patch_norm		,       layer_norm_eps=self.layer_norm_eps		,       initializer_range=self.initializer_range		,       encoder_stride=self.encoder_stride		,       )




	def    __UpperCAmelCase							(     self		,       __a		,       __a		,       __a							):

						'''simple docstring'''





						__a    : Any      					=						TFSwinModel(config=__a							)
						__a    : List[str]      					=						model(__a							)

						__a    : Dict      					=						((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths							) - 1))
						__a    : Any      					=						int(config.embed_dim * 2 ** (len(config.depths							) - 1)							)

						self.parent.assertEqual(result.last_hidden_state.shape		,       (self.batch_size, expected_seq_len, expected_dim)							)




	def    __UpperCAmelCase							(     self		,       __a		,       __a		,       __a							):

						'''simple docstring'''





						__a    : Any      					=						TFSwinForMaskedImageModeling(config=__a							)
						__a    : Tuple      					=						model(__a							)
						self.parent.assertEqual(
						    result.logits.shape		,       (self.batch_size, self.num_channels, self.image_size, self.image_size)							)

						# test greyscale images
						__a    : str      					=						1
						__a    : Union[str, Any]      					=						TFSwinForMaskedImageModeling(__a							)

						__a    : Optional[int]      					=						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
						__a    : Any      					=						model(__a							)
						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, 1, self.image_size, self.image_size)							)




	def    __UpperCAmelCase							(     self		,       __a		,       __a		,       __a							):

						'''simple docstring'''





						__a    : Dict      					=						self.type_sequence_label_size
						__a    : Tuple      					=						TFSwinForImageClassification(__a							)
						__a    : Any      					=						model(__a		,       labels=__a							)
						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, self.type_sequence_label_size)							)

						# test greyscale images
						__a    : Tuple      					=						1
						__a    : str      					=						TFSwinForImageClassification(__a							)
						__a    : List[Any]      					=						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
						__a    : List[str]      					=						model(__a							)
						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, self.type_sequence_label_size)							)





	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a    : List[Any]      					=						self.prepare_config_and_inputs()
						__a					,       __a					,       __a    : Tuple      					=						config_and_inputs
						__a    : Dict      					=						{'pixel_values': pixel_values}
						return config, inputs_dict





@require_tf
class 							__UpperCamelCase       (	lowerCAmelCase_	,   lowerCAmelCase_	,   unittest.TestCase      ):
	A_			    =					(
	    (
	        TFSwinModel,
	        TFSwinForImageClassification,
	        TFSwinForMaskedImageModeling,
	    )
	    if is_tf_available()
	    else ()
	)
	A_			    =					(
	    {"feature-extraction": TFSwinModel, "image-classification": TFSwinForImageClassification}
	    if is_tf_available()
	    else {}
	)

	A_			    =					False
	A_			    =					False
	A_			    =					False
	A_			    =					False




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a    : Optional[int]      					=						TFSwinModelTester(self							)
						__a    : Union[str, Any]      					=						ConfigTester(self		,       config_class=__a		,       embed_dim=37							)




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						self.create_and_test_config_common_properties()
						self.config_tester.create_and_test_config_to_json_string()
						self.config_tester.create_and_test_config_to_json_file()
						self.config_tester.create_and_test_config_from_and_save_pretrained()
						self.config_tester.create_and_test_config_with_num_labels()
						self.config_tester.check_config_can_be_init_without_params()
						self.config_tester.check_config_arguments_init()




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						return




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a    : List[str]      					=						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*__a							)




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a    : List[Any]      					=						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_image_modeling(*__a							)




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a    : Optional[Any]      					=						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_image_classification(*__a							)




	@unittest.skip(reason='Swin does not use inputs_embeds'							)
	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						pass




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a					,       __a    : int      					=						self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											__a    : Dict      					=						model_class(__a							)
											self.assertIsInstance(model.get_input_embeddings()		,       tf.keras.layers.Layer							)
											__a    : str      					=						model.get_output_embeddings()
											self.assertTrue(x is None or isinstance(__a		,       tf.keras.layers.Dense							)							)




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a					,       __a    : Dict      					=						self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											__a    : Optional[int]      					=						model_class(__a							)
											__a    : Union[str, Any]      					=						inspect.signature(model.call							)
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											__a    : Optional[int]      					=						[*signature.parameters.keys()]

											__a    : str      					=						['pixel_values']
											self.assertListEqual(arg_names[:1]		,       __a							)




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a					,       __a    : Any      					=						self.model_tester.prepare_config_and_inputs_for_common()
						__a    : Optional[int]      					=						True

						for model_class in self.all_model_classes:
											__a    : Optional[int]      					=						True
											__a    : Dict      					=						False
											__a    : Any      					=						True
											__a    : Tuple      					=						model_class(__a							)
											__a    : List[str]      					=						model(**self._prepare_for_class(__a		,       __a							)							)
											__a    : int      					=						outputs.attentions
											__a    : List[Any]      					=						len(self.model_tester.depths							)
											self.assertEqual(len(__a							)		,       __a							)

											# check that output_attentions also work using config
											del inputs_dict["output_attentions"]
											__a    : Any      					=						True
											__a    : List[Any]      					=						config.window_size**2
											__a    : Optional[Any]      					=						model_class(__a							)
											__a    : Tuple      					=						model(**self._prepare_for_class(__a		,       __a							)							)
											__a    : str      					=						outputs.attentions
											self.assertEqual(len(__a							)		,       __a							)

											self.assertListEqual(
											    list(attentions[0].shape[-3:]							)		,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]		,       )
											__a    : Any      					=						len(__a							)

											# Check attention is always last and order is fine
											__a    : int      					=						True
											__a    : Any      					=						True
											__a    : str      					=						model_class(__a							)
											__a    : str      					=						model(**self._prepare_for_class(__a		,       __a							)							)

											if hasattr(self.model_tester		,       'num_hidden_states_types'							):
																__a    : Optional[int]      					=						self.model_tester.num_hidden_states_types
											else:
																# also another +1 for reshaped_hidden_states
																__a    : Union[str, Any]      					=						2
											self.assertEqual(out_len + added_hidden_states		,       len(__a							)							)

											__a    : Dict      					=						outputs.attentions

											self.assertEqual(len(__a							)		,       __a							)

											self.assertListEqual(
											    list(self_attentions[0].shape[-3:]							)		,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]		,       )




	def    __UpperCAmelCase							(     self		,       __a		,       __a		,       __a		,       __a							):

						'''simple docstring'''





						__a    : Optional[Any]      					=						model_class(__a							)
						__a    : int      					=						model(**self._prepare_for_class(__a		,       __a							)							)
						__a    : Tuple      					=						outputs.hidden_states

						__a    : Any      					=						getattr(
						    self.model_tester		,       'expected_num_hidden_layers'		,       len(self.model_tester.depths							) + 1							)
						self.assertEqual(len(__a							)		,       __a							)

						# Swin has a different seq_length
						__a    : Optional[int]      					=						to_atuple(config.patch_size							)

						__a    : List[str]      					=						(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

						self.assertListEqual(
						    list(hidden_states[0].shape[-2:]							)		,       [num_patches, self.model_tester.embed_dim]		,       )

						__a    : str      					=						outputs.reshaped_hidden_states
						self.assertEqual(len(__a							)		,       __a							)

						__a					,       __a					,       __a					,       __a    : str      					=						reshaped_hidden_states[0].shape

						__a    : int      					=						tf.reshape(reshaped_hidden_states[0]		,       (batch_size, num_channels, height * width)							)
						__a    : List[str]      					=						tf.transpose(__a		,       (0, 2, 1)							)

						self.assertListEqual(
						    list(reshaped_hidden_states.shape[-2:]							)		,       [num_patches, self.model_tester.embed_dim]		,       )




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a					,       __a    : Optional[Any]      					=						self.model_tester.prepare_config_and_inputs_for_common()

						__a    : Union[str, Any]      					=						to_atuple(self.model_tester.image_size							)

						for model_class in self.all_model_classes:
											__a    : Optional[int]      					=						True
											self.check_hidden_states_output(__a		,       __a		,       __a		,       __a							)

											# check that output_hidden_states also work using config
											del inputs_dict["output_hidden_states"]
											__a    : Tuple      					=						True

											self.check_hidden_states_output(__a		,       __a		,       __a		,       __a							)




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a					,       __a    : List[str]      					=						self.model_tester.prepare_config_and_inputs_for_common()
						__a    : List[str]      					=						3

						__a    : Any      					=						to_atuple(self.model_tester.image_size							)
						__a    : Any      					=						to_atuple(config.patch_size							)

						__a    : Any      					=						image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
						__a    : Optional[int]      					=						image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

						for model_class in self.all_model_classes:
											__a    : Union[str, Any]      					=						True
											self.check_hidden_states_output(__a		,       __a		,       __a		,       (padded_height, padded_width)							)

											# check that output_hidden_states also work using config
											del inputs_dict["output_hidden_states"]
											__a    : int      					=						True
											self.check_hidden_states_output(__a		,       __a		,       __a		,       (padded_height, padded_width)							)





	@slow
	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						for model_name in TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__a    : List[Any]      					=						TFSwinModel.from_pretrained(__a							)
											self.assertIsNotNone(__a							)





@require_vision
@require_tf
class 							__UpperCamelCase       (	unittest.TestCase      ):




	@cached_property
	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						return (
						    AutoImageProcessor.from_pretrained('microsoft/swin-tiny-patch4-window7-224'							)
						    if is_vision_available()
						    else None
						)





	@slow
	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						__a    : Optional[Any]      					=						TFSwinForImageClassification.from_pretrained('microsoft/swin-tiny-patch4-window7-224'							)
						__a    : str      					=						self.default_image_processor

						__a    : Optional[Any]      					=						Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'							)
						__a    : Optional[Any]      					=						image_processor(images=__a		,       return_tensors='tf'							)

						# forward pass
						__a    : List[Any]      					=						model(__a							)

						# verify the logits
						__a    : List[str]      					=						tf.TensorShape((1, 1000)							)
						self.assertEqual(outputs.logits.shape		,       __a							)
						__a    : Optional[Any]      					=						tf.constant([-0.0948, -0.6454, -0.0921]							)
						self.assertTrue(np.allclose(outputs.logits[0, :3]		,       __a		,       atol=1E-4							)							)



