code: 
  
  
  import os
  from argparse import ArgumentParser
  from pathlib import Path
  from shutil import copyfile
  from typing import List
  
  from datasets import config
  from datasets.builder import DatasetBuilder
  from datasets.commands import BaseDatasetsCLICommand
  from datasets.download.download_config import DownloadConfig
  from datasets.download.download_manager import DownloadMode
  from datasets.load import dataset_module_factory, import_main_class
  from datasets.utils.info_utils import VerificationMode
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase							,     **_UpperCAmelCase				):
  						return RunBeamCommand(
  						    args.dataset							,     args.name							,     args.cache_dir							,     args.beam_pipeline_options							,     args.data_dir							,     args.all_configs							,     args.save_info or args.save_infos							,     args.ignore_verifications							,     args.force_redownload							,     **_UpperCAmelCase							,     )
  
  
  
  
  
  
  class 					_UpperCamelCase (      lowerCamelCase__	):
  
  
  
  
  
  
  
  							'''simple docstring'''
  
  
  
  
  
  
  
  							@staticmethod
  							def 		_UpperCAmelCase	(		SCREAMING_SNAKE_CASE_  :				ArgumentParser		):
  													_a     =			parser.add_parser('run_beam'		,				help='Run a Beam dataset processing pipeline'		)
  													run_beam_parser.add_argument('dataset'		,				type=SCREAMING_SNAKE_CASE_		,				help='Name of the dataset to download'		)
  													run_beam_parser.add_argument('--name'		,				type=SCREAMING_SNAKE_CASE_		,				default=SCREAMING_SNAKE_CASE_		,				help='Dataset config name'		)
  													run_beam_parser.add_argument(
  													    '--cache_dir'		,				type=SCREAMING_SNAKE_CASE_		,				default=SCREAMING_SNAKE_CASE_		,				help='Cache directory where the datasets are stored'		,				)
  													run_beam_parser.add_argument(
  													    '--beam_pipeline_options'		,				type=SCREAMING_SNAKE_CASE_		,				default=''		,				help='Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`'		,				)
  													run_beam_parser.add_argument(
  													    '--data_dir'		,				type=SCREAMING_SNAKE_CASE_		,				default=SCREAMING_SNAKE_CASE_		,				help='Can be used to specify a manual directory to get the files from'		,				)
  													run_beam_parser.add_argument('--all_configs'		,				action='store_true'		,				help='Test all dataset configurations'		)
  													run_beam_parser.add_argument('--save_info'		,				action='store_true'		,				help='Save the dataset infos file'		)
  													run_beam_parser.add_argument(
  													    '--ignore_verifications'		,				action='store_true'		,				help='Run the test without checksums and splits checks'		)
  													run_beam_parser.add_argument('--force_redownload'		,				action='store_true'		,				help='Force dataset redownload'		)
  													# aliases
  													run_beam_parser.add_argument('--save_infos'		,				action='store_true'		,				help='alias for save_info'		)
  													run_beam_parser.set_defaults(func=SCREAMING_SNAKE_CASE_		)
  
  
  
  
  
  
  
  							def __init__(		self  :				Optional[Any]		,				SCREAMING_SNAKE_CASE_  :				str		,				SCREAMING_SNAKE_CASE_  :				str		,				SCREAMING_SNAKE_CASE_  :				str		,				SCREAMING_SNAKE_CASE_  :				str		,				SCREAMING_SNAKE_CASE_  :				str		,				SCREAMING_SNAKE_CASE_  :				bool		,				SCREAMING_SNAKE_CASE_  :				bool		,				SCREAMING_SNAKE_CASE_  :				bool		,				SCREAMING_SNAKE_CASE_  :				bool		,				**SCREAMING_SNAKE_CASE_  :				Optional[int]		,				):
  													_a     =			dataset
  													_a     =			name
  													_a     =			cache_dir
  													_a     =			beam_pipeline_options
  													_a     =			data_dir
  													_a     =			all_configs
  													_a     =			save_infos
  													_a     =			ignore_verifications
  													_a     =			force_redownload
  													_a     =			config_kwargs
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Any		):
  													import apache_beam as beam
  
  													if self._name is not None and self._all_configs:
  																			print('Both parameters `name` and `all_configs` can\'t be used at once.'		)
  																			exit(1		)
  													_a ,	_a     =			self._dataset, self._name
  													_a     =			dataset_module_factory(SCREAMING_SNAKE_CASE_		)
  													_a     =			import_main_class(dataset_module.module_path		)
  													_a     =			[]
  													if self._beam_pipeline_options:
  																			_a     =			beam.options.pipeline_options.PipelineOptions(
  																			    flags=[f"""--{opt.strip()}""" for opt in self._beam_pipeline_options.split(','		) if opt]		)
  													else:
  																			_a     =			None
  													if self._all_configs and len(builder_cls.BUILDER_CONFIGS		) > 0:
  																			for builder_config in builder_cls.BUILDER_CONFIGS:
  																									builders.append(
  																									    builder_cls(
  																									        config_name=builder_config.name		,				data_dir=self._data_dir		,				hash=dataset_module.hash		,				beam_options=SCREAMING_SNAKE_CASE_		,				cache_dir=self._cache_dir		,				base_path=dataset_module.builder_kwargs.get('base_path'		)		,				)		)
  													else:
  																			builders.append(
  																			    builder_cls(
  																			        config_name=SCREAMING_SNAKE_CASE_		,				data_dir=self._data_dir		,				beam_options=SCREAMING_SNAKE_CASE_		,				cache_dir=self._cache_dir		,				base_path=dataset_module.builder_kwargs.get('base_path'		)		,				**self._config_kwargs		,				)		)
  
  													for builder in builders:
  																			builder.download_and_prepare(
  																			    download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
  																			    if not self._force_redownload
  																			    else DownloadMode.FORCE_REDOWNLOAD		,				download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH		)		,				verification_mode=VerificationMode.NO_CHECKS
  																			    if self._ignore_verifications
  																			    else VerificationMode.ALL_CHECKS		,				try_from_hf_gcs=SCREAMING_SNAKE_CASE_		,				)
  																			if self._save_infos:
  																									builder._save_infos()
  
  													print('Apache beam run successful.'		)
  
  													# If save_infos=True, the dataset infos file is created next to the loaded module file.
  													# Let's move it to the original directory of the dataset script, to allow the user to
  													# upload them on S3 at the same time afterwards.
  													if self._save_infos:
  																			_a     =			os.path.join(builder_cls.get_imported_module_dir()		,				config.DATASETDICT_INFOS_FILENAME		)
  
  																			_a     =			Path(SCREAMING_SNAKE_CASE_		).name + '.py'
  
  																			_a     =			os.path.join(SCREAMING_SNAKE_CASE_		,				SCREAMING_SNAKE_CASE_		)
  																			if os.path.isfile(SCREAMING_SNAKE_CASE_		):
  																									_a     =			os.path.dirname(SCREAMING_SNAKE_CASE_		)
  																			elif os.path.isfile(SCREAMING_SNAKE_CASE_		):
  																									_a     =			path
  																			else:  # in case of a remote dataset
  																									print(f"""Dataset Infos file saved at {dataset_infos_path}"""		)
  																									exit(1		)
  
  																			# Move datasetinfo back to the user
  																			_a     =			os.path.join(SCREAMING_SNAKE_CASE_		,				config.DATASETDICT_INFOS_FILENAME		)
  																			copyfile(SCREAMING_SNAKE_CASE_		,				SCREAMING_SNAKE_CASE_		)
  																			print(f"""Dataset Infos file saved at {user_dataset_infos_path}"""		)
  
  
  
style_context: 
  
  
  from typing import List, Union
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  lowercase_								=    logging.get_logger(__name__)
  
  lowercase_								=    {
      'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/config.json',
      # See all LED models at https://huggingface.co/models?filter=led
  }
  
  
  
  
  class 					_UpperCamelCase (      lowerCamelCase__	):
  
  
  
  
  
  
  
  							'''simple docstring'''
  							_A  				=  "led"
  							_A  				=  {
  							    "num_attention_heads": "encoder_attention_heads",
  							    "hidden_size": "d_model",
  							    "attention_probs_dropout_prob": "attention_dropout",
  							    "initializer_range": "init_std",
  							}
  
  
  
  
  
  
  
  							def __init__(		self  :				List[str]		,				SCREAMING_SNAKE_CASE_  :				List[str]=5_0_2_6_5		,				SCREAMING_SNAKE_CASE_  :				str=1_6_3_8_4		,				SCREAMING_SNAKE_CASE_  :				List[Any]=1_0_2_4		,				SCREAMING_SNAKE_CASE_  :				List[str]=1_2		,				SCREAMING_SNAKE_CASE_  :				Any=4_0_9_6		,				SCREAMING_SNAKE_CASE_  :				Optional[Any]=1_6		,				SCREAMING_SNAKE_CASE_  :				str=1_2		,				SCREAMING_SNAKE_CASE_  :				List[Any]=4_0_9_6		,				SCREAMING_SNAKE_CASE_  :				List[str]=1_6		,				SCREAMING_SNAKE_CASE_  :				Optional[int]=0.0		,				SCREAMING_SNAKE_CASE_  :				Union[str, Any]=0.0		,				SCREAMING_SNAKE_CASE_  :				List[Any]=True		,				SCREAMING_SNAKE_CASE_  :				str=True		,				SCREAMING_SNAKE_CASE_  :				List[str]="gelu"		,				SCREAMING_SNAKE_CASE_  :				List[Any]=1_0_2_4		,				SCREAMING_SNAKE_CASE_  :				Tuple=0.1		,				SCREAMING_SNAKE_CASE_  :				Dict=0.0		,				SCREAMING_SNAKE_CASE_  :				Optional[Any]=0.0		,				SCREAMING_SNAKE_CASE_  :				Tuple=0.02		,				SCREAMING_SNAKE_CASE_  :				str=2		,				SCREAMING_SNAKE_CASE_  :				Tuple=0.0		,				SCREAMING_SNAKE_CASE_  :				Optional[int]=1		,				SCREAMING_SNAKE_CASE_  :				str=0		,				SCREAMING_SNAKE_CASE_  :				Union[str, Any]=2		,				SCREAMING_SNAKE_CASE_  :				Union[List[int], int] = 5_1_2		,				**SCREAMING_SNAKE_CASE_  :				Dict		,				):
  													_a     =			vocab_size
  													_a     =			max_encoder_position_embeddings
  													_a     =			max_decoder_position_embeddings
  													_a     =			d_model
  													_a     =			encoder_ffn_dim
  													_a     =			encoder_layers
  													_a     =			encoder_attention_heads
  													_a     =			decoder_ffn_dim
  													_a     =			decoder_layers
  													_a     =			decoder_attention_heads
  													_a     =			dropout
  													_a     =			attention_dropout
  													_a     =			activation_dropout
  													_a     =			activation_function
  													_a     =			init_std
  													_a     =			encoder_layerdrop
  													_a     =			decoder_layerdrop
  													_a     =			classifier_dropout
  													_a     =			use_cache
  													_a     =			encoder_layers
  													_a     =			attention_window
  
  													super().__init__(
  													    pad_token_id=SCREAMING_SNAKE_CASE_		,				bos_token_id=SCREAMING_SNAKE_CASE_		,				eos_token_id=SCREAMING_SNAKE_CASE_		,				is_encoder_decoder=SCREAMING_SNAKE_CASE_		,				decoder_start_token_id=SCREAMING_SNAKE_CASE_		,				**SCREAMING_SNAKE_CASE_		,				)
  
  
  