


'''simple docstring'''


import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


lowerCamelCase__					=   logging.getLogger(__name__)

lowerCamelCase__					=   tf.data.AUTOTUNE






def        __lowerCAmelCase       ():
		_UpperCAmelCase						:				Any	    =					argparse.ArgumentParser(description="Train a masked language model on TPU."      )
		parser.add_argument(
		    "--pretrained_model_config"					,       type=__lowerCAmelCase					,       default="roberta-base"					,       help="The model config to use. Note that we don't copy the model's weights, only the config!"					,       )
		parser.add_argument(
		    "--tokenizer"					,       type=__lowerCAmelCase					,       default="unigram-tokenizer-wikitext"					,       help="The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."					,       )

		parser.add_argument(
		    "--per_replica_batch_size"					,       type=__lowerCAmelCase					,       default=8					,       help="Batch size per TPU core."					,       )

		parser.add_argument(
		    "--no_tpu"					,       action="store_true"					,       help="If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."					,       )

		parser.add_argument(
		    "--tpu_name"					,       type=__lowerCAmelCase					,       help="Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."					,       default="local"					,       )

		parser.add_argument(
		    "--tpu_zone"					,       type=__lowerCAmelCase					,       help="Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."					,       )

		parser.add_argument(
		    "--gcp_project"					,       type=__lowerCAmelCase					,       help="Google cloud project name. Only used for non-Colab TPU nodes."      )

		parser.add_argument(
		    "--bfloat16"					,       action="store_true"					,       help="Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."					,       )

		parser.add_argument(
		    "--train_dataset"					,       type=__lowerCAmelCase					,       help="Path to training dataset to load. If the path begins with `gs://`"
		    " then the dataset will be loaded from a Google Cloud Storage bucket."					,       )

		parser.add_argument(
		    "--shuffle_buffer_size"					,       type=__lowerCAmelCase					,       default=2**18					,       help="Size of the shuffle buffer (in samples)"					,       )

		parser.add_argument(
		    "--eval_dataset"					,       type=__lowerCAmelCase					,       help="Path to evaluation dataset to load. If the path begins with `gs://`"
		    " then the dataset will be loaded from a Google Cloud Storage bucket."					,       )

		parser.add_argument(
		    "--num_epochs"					,       type=__lowerCAmelCase					,       default=1					,       help="Number of epochs to train for."					,       )

		parser.add_argument(
		    "--learning_rate"					,       type=__lowerCAmelCase					,       default=1e-4					,       help="Learning rate to use for training."					,       )

		parser.add_argument(
		    "--weight_decay_rate"					,       type=__lowerCAmelCase					,       default=1e-3					,       help="Weight decay rate to use for training."					,       )

		parser.add_argument(
		    "--max_length"					,       type=__lowerCAmelCase					,       default=512					,       help="Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"					,       )

		parser.add_argument(
		    "--mlm_probability"					,       type=__lowerCAmelCase					,       default=0.1_5					,       help="Fraction of tokens to mask during training."					,       )

		parser.add_argument("--output_dir"					,       type=__lowerCAmelCase					,       required=__lowerCAmelCase					,       help="Path to save model checkpoints to."      )
		parser.add_argument("--hub_model_id"					,       type=__lowerCAmelCase					,       help="Model ID to upload to on the Hugging Face Hub."      )

		_UpperCAmelCase						:				Any	    =					parser.parse_args()
		return args






def        __lowerCAmelCase       (__lowerCAmelCase      ):
		try:
				if args.tpu_name:
						_UpperCAmelCase						:				List[str]	    =					tf.distribute.cluster_resolver.TPUClusterResolver(
						    args.tpu_name					,       zone=args.tpu_zone					,       project=args.gcp_project      )
				else:
						_UpperCAmelCase						:				List[Any]	    =					tf.distribute.cluster_resolver.TPUClusterResolver()
		except ValueError:
				raise RuntimeError(
				    "Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or "
				    "--gcp_project. When running on a TPU VM, use --tpu_name local."      )

		tf.config.experimental_connect_to_cluster(__lowerCAmelCase      )
		tf.tpu.experimental.initialize_tpu_system(__lowerCAmelCase      )

		return tpu






def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Union[str, Any]	    =					0
		for file in file_list:
				_UpperCAmelCase						:				Union[str, Any]	    =					file.split("/"      )[-1]
				_UpperCAmelCase						:				int	    =					re.search(R"-\d+-(\d+)\.tfrecord"					,       __lowerCAmelCase      ).group(1      )
				_UpperCAmelCase						:				Any	    =					int(__lowerCAmelCase      )
				num_samples += sample_count

		return num_samples






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase=None      ):
		_UpperCAmelCase						:				str	    =					count_samples(__lowerCAmelCase      )
		_UpperCAmelCase						:				Tuple	    =					tf.data.Dataset.from_tensor_slices(__lowerCAmelCase      )
		if shuffle:
				_UpperCAmelCase						:				List[str]	    =					dataset.shuffle(len(__lowerCAmelCase      )      )
		_UpperCAmelCase						:				Union[str, Any]	    =					tf.data.TFRecordDataset(__lowerCAmelCase					,       num_parallel_reads=__lowerCAmelCase      )
		# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
		_UpperCAmelCase						:				List[str]	    =					dataset.apply(tf.data.experimental.assert_cardinality(__lowerCAmelCase      )      )
		_UpperCAmelCase						:				Dict	    =					dataset.map(__lowerCAmelCase					,       num_parallel_calls=__lowerCAmelCase      )
		if shuffle:
				assert shuffle_buffer_size is not None
				_UpperCAmelCase						:				int	    =					dataset.shuffle(args.shuffle_buffer_size      )
		_UpperCAmelCase						:				int	    =					dataset.batch(__lowerCAmelCase					,       drop_remainder=__lowerCAmelCase      )
		_UpperCAmelCase						:				Union[str, Any]	    =					dataset.map(__lowerCAmelCase					,       num_parallel_calls=__lowerCAmelCase      )
		_UpperCAmelCase						:				Optional[Any]	    =					dataset.prefetch(__lowerCAmelCase      )
		return dataset






def        __lowerCAmelCase       (__lowerCAmelCase      ):
		if not args.no_tpu:
				_UpperCAmelCase						:				Dict	    =					initialize_tpu(__lowerCAmelCase      )
				_UpperCAmelCase						:				List[str]	    =					tf.distribute.TPUStrategy(__lowerCAmelCase      )
		else:
				_UpperCAmelCase						:				Tuple	    =					tf.distribute.OneDeviceStrategy(device="/gpu:0"      )

		if args.bfloataa:
				tf.keras.mixed_precision.set_global_policy("mixed_bfloat16"      )

		_UpperCAmelCase						:				str	    =					AutoTokenizer.from_pretrained(args.tokenizer      )
		_UpperCAmelCase						:				List[str]	    =					AutoConfig.from_pretrained(args.pretrained_model_config      )
		_UpperCAmelCase						:				Union[str, Any]	    =					tokenizer.vocab_size

		_UpperCAmelCase						:				Dict	    =					tf.io.gfile.glob(os.path.join(args.train_dataset					,       "*.tfrecord"      )      )
		if not training_records:
				raise ValueError(F"""No .tfrecord files found in {args.train_dataset}."""      )
		_UpperCAmelCase						:				List[Any]	    =					tf.io.gfile.glob(os.path.join(args.eval_dataset					,       "*.tfrecord"      )      )
		if not eval_records:
				raise ValueError(F"""No .tfrecord files found in {args.eval_dataset}."""      )

		_UpperCAmelCase						:				int	    =					count_samples(__lowerCAmelCase      )

		_UpperCAmelCase						:				List[Any]	    =					num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
		_UpperCAmelCase						:				Tuple	    =					steps_per_epoch * args.num_epochs

		with strategy.scope():
				_UpperCAmelCase						:				Dict	    =					TFAutoModelForMaskedLM.from_config(__lowerCAmelCase      )
				model(model.dummy_inputs      )  # Pass some dummy inputs through the model to ensure all the weights are built
				_UpperCAmelCase    ,			_UpperCAmelCase						:				str	    =					create_optimizer(
				    num_train_steps=__lowerCAmelCase					,       num_warmup_steps=total_train_steps // 20					,       init_lr=args.learning_rate					,       weight_decay_rate=args.weight_decay_rate					,       )

				# Transformers models compute the right loss for their task by default when labels are passed, and will
				# use this for training unless you specify your own loss function in compile().
				model.compile(optimizer=__lowerCAmelCase					,       metrics=["accuracy"]      )

		def decode_fn(__lowerCAmelCase      ):
				_UpperCAmelCase						:				Any	    =					{
				    "input_ids": tf.io.FixedLenFeature(dtype=tf.intaa					,       shape=(args.max_length,)      ),
				    "attention_mask": tf.io.FixedLenFeature(dtype=tf.intaa					,       shape=(args.max_length,)      ),
				}
				return tf.io.parse_single_example(__lowerCAmelCase					,       __lowerCAmelCase      )

		# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
		# use their methods in our data pipeline.
		_UpperCAmelCase						:				List[str]	    =					DataCollatorForLanguageModeling(
		    tokenizer=__lowerCAmelCase					,       mlm_probability=args.mlm_probability					,       mlm=__lowerCAmelCase					,       return_tensors="tf"      )

		def mask_with_collator(__lowerCAmelCase      ):
				# TF really needs an isin() function
				_UpperCAmelCase						:				int	    =					(
				    ~tf.cast(batch["attention_mask"]					,       tf.bool      )
				    | (batch["input_ids"] == tokenizer.cls_token_id)
				    | (batch["input_ids"] == tokenizer.sep_token_id)
				)
				_UpperCAmelCase    ,			_UpperCAmelCase						:				Any	    =					data_collator.tf_mask_tokens(
				    batch["input_ids"]					,       vocab_size=len(__lowerCAmelCase      )					,       mask_token_id=tokenizer.mask_token_id					,       special_tokens_mask=__lowerCAmelCase					,       )
				return batch

		_UpperCAmelCase						:				Optional[int]	    =					args.per_replica_batch_size * strategy.num_replicas_in_sync

		_UpperCAmelCase						:				List[str]	    =					prepare_dataset(
		    __lowerCAmelCase					,       decode_fn=__lowerCAmelCase					,       mask_fn=__lowerCAmelCase					,       batch_size=__lowerCAmelCase					,       shuffle=__lowerCAmelCase					,       shuffle_buffer_size=args.shuffle_buffer_size					,       )

		_UpperCAmelCase						:				List[Any]	    =					prepare_dataset(
		    __lowerCAmelCase					,       decode_fn=__lowerCAmelCase					,       mask_fn=__lowerCAmelCase					,       batch_size=__lowerCAmelCase					,       shuffle=__lowerCAmelCase					,       )

		_UpperCAmelCase						:				Tuple	    =					[]
		if args.hub_model_id:
				callbacks.append(
				    PushToHubCallback(output_dir=args.output_dir					,       hub_model_id=args.hub_model_id					,       tokenizer=__lowerCAmelCase      )      )

		model.fit(
		    __lowerCAmelCase					,       validation_data=__lowerCAmelCase					,       epochs=args.num_epochs					,       callbacks=__lowerCAmelCase					,       )

		model.save_pretrained(args.output_dir      )


if __name__ == "__main__":
						lowerCamelCase__					=   parse_args()
						main(args)







'''simple docstring'''


import argparse
import re
from pathlib import Path

import requests
import torch
from PIL import Image
from torchvision.transforms import CenterCrop, Compose, Normalize, Resize, ToTensor

from transformers import (
    EfficientFormerConfig,
    EfficientFormerForImageClassificationWithTeacher,
    EfficientFormerImageProcessor,
)
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Union[str, Any]	    =					old_name

		if "patch_embed" in old_name:
				_UpperCAmelCase    ,			_UpperCAmelCase    ,			_UpperCAmelCase						:				Dict	    =					old_name.split("."      )

				if layer == "0":
						_UpperCAmelCase						:				List[str]	    =					old_name.replace("0"					,       "convolution1"      )
				elif layer == "1":
						_UpperCAmelCase						:				Dict	    =					old_name.replace("1"					,       "batchnorm_before"      )
				elif layer == "3":
						_UpperCAmelCase						:				Tuple	    =					old_name.replace("3"					,       "convolution2"      )
				else:
						_UpperCAmelCase						:				Tuple	    =					old_name.replace("4"					,       "batchnorm_after"      )

		if "network" in old_name and re.search(R"\d\.\d"					,       __lowerCAmelCase      ):
				_UpperCAmelCase						:				List[Any]	    =					R"\b\d{2}\b"
				if bool(re.search(__lowerCAmelCase					,       __lowerCAmelCase      )      ):
						_UpperCAmelCase						:				Optional[int]	    =					re.search(R"\d\.\d\d."					,       __lowerCAmelCase      ).group()
				else:
						_UpperCAmelCase						:				Any	    =					re.search(R"\d\.\d."					,       __lowerCAmelCase      ).group()
				if int(match[0]      ) < 6:
						_UpperCAmelCase						:				str	    =					old_name.replace(__lowerCAmelCase					,       ""      )
						_UpperCAmelCase						:				Optional[Any]	    =					trimmed_name.replace("network"					,       match[0] + ".meta4D_layers.blocks." + match[2:-1]      )
						_UpperCAmelCase						:				Union[str, Any]	    =					"intermediate_stages." + trimmed_name
				else:
						_UpperCAmelCase						:				Tuple	    =					old_name.replace(__lowerCAmelCase					,       ""      )
						if int(match[2]      ) < num_meta4D_last_stage:
								_UpperCAmelCase						:				Any	    =					trimmed_name.replace("network"					,       "meta4D_layers.blocks." + match[2]      )
						else:
								_UpperCAmelCase						:				List[str]	    =					str(int(match[2]      ) - num_meta4D_last_stage      )
								_UpperCAmelCase						:				int	    =					trimmed_name.replace("network"					,       "meta3D_layers.blocks." + layer_index      )
								if "norm1" in old_name:
										_UpperCAmelCase						:				Tuple	    =					trimmed_name.replace("norm1"					,       "layernorm1"      )
								elif "norm2" in old_name:
										_UpperCAmelCase						:				int	    =					trimmed_name.replace("norm2"					,       "layernorm2"      )
								elif "fc1" in old_name:
										_UpperCAmelCase						:				Optional[int]	    =					trimmed_name.replace("fc1"					,       "linear_in"      )
								elif "fc2" in old_name:
										_UpperCAmelCase						:				List[str]	    =					trimmed_name.replace("fc2"					,       "linear_out"      )

						_UpperCAmelCase						:				Optional[Any]	    =					"last_stage." + trimmed_name

		elif "network" in old_name and re.search(R".\d."					,       __lowerCAmelCase      ):
				_UpperCAmelCase						:				Optional[Any]	    =					old_name.replace("network"					,       "intermediate_stages"      )

		if "fc" in new_name:
				_UpperCAmelCase						:				Union[str, Any]	    =					new_name.replace("fc"					,       "convolution"      )
		elif ("norm1" in new_name) and ("layernorm1" not in new_name):
				_UpperCAmelCase						:				List[Any]	    =					new_name.replace("norm1"					,       "batchnorm_before"      )
		elif ("norm2" in new_name) and ("layernorm2" not in new_name):
				_UpperCAmelCase						:				List[Any]	    =					new_name.replace("norm2"					,       "batchnorm_after"      )
		if "proj" in new_name:
				_UpperCAmelCase						:				Union[str, Any]	    =					new_name.replace("proj"					,       "projection"      )
		if "dist_head" in new_name:
				_UpperCAmelCase						:				List[Any]	    =					new_name.replace("dist_head"					,       "distillation_classifier"      )
		elif "head" in new_name:
				_UpperCAmelCase						:				str	    =					new_name.replace("head"					,       "classifier"      )
		elif "patch_embed" in new_name:
				_UpperCAmelCase						:				List[str]	    =					"efficientformer." + new_name
		elif new_name == "norm.weight" or new_name == "norm.bias":
				_UpperCAmelCase						:				List[Any]	    =					new_name.replace("norm"					,       "layernorm"      )
				_UpperCAmelCase						:				Any	    =					"efficientformer." + new_name
		else:
				_UpperCAmelCase						:				Dict	    =					"efficientformer.encoder." + new_name

		return new_name






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		for key in checkpoint.copy().keys():
				_UpperCAmelCase						:				List[Any]	    =					checkpoint.pop(__lowerCAmelCase      )
				_UpperCAmelCase						:				List[Any]	    =					val

		return checkpoint






def        __lowerCAmelCase       ():
		_UpperCAmelCase						:				Optional[int]	    =					"http://images.cocodataset.org/val2017/000000039769.jpg"
		_UpperCAmelCase						:				Tuple	    =					Image.open(requests.get(__lowerCAmelCase					,       stream=__lowerCAmelCase      ).raw      )

		return image






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Union[str, Any]	    =					torch.load(__lowerCAmelCase					,       map_location="cpu"      )["model"]
		_UpperCAmelCase						:				Dict	    =					EfficientFormerConfig.from_json_file(__lowerCAmelCase      )
		_UpperCAmelCase						:				Optional[Any]	    =					EfficientFormerForImageClassificationWithTeacher(__lowerCAmelCase      )
		_UpperCAmelCase						:				Tuple	    =					"_".join(checkpoint_path.split("/"      )[-1].split("."      )[0].split("_"      )[:-1]      )

		_UpperCAmelCase						:				Union[str, Any]	    =					config.depths[-1] - config.num_metaad_blocks + 1
		_UpperCAmelCase						:				Optional[int]	    =					convert_torch_checkpoint(__lowerCAmelCase					,       __lowerCAmelCase      )

		model.load_state_dict(__lowerCAmelCase      )
		model.eval()

		_UpperCAmelCase						:				Optional[Any]	    =					{
		    "bilinear": PILImageResampling.BILINEAR,
		    "bicubic": PILImageResampling.BICUBIC,
		    "nearest": PILImageResampling.NEAREST,
		}

		# prepare image
		_UpperCAmelCase						:				int	    =					prepare_img()
		_UpperCAmelCase						:				List[str]	    =					256
		_UpperCAmelCase						:				Optional[int]	    =					224
		_UpperCAmelCase						:				Tuple	    =					EfficientFormerImageProcessor(
		    size={"shortest_edge": image_size}					,       crop_size={"height": crop_size, "width": crop_size}					,       resample=pillow_resamplings["bicubic"]					,       )
		_UpperCAmelCase						:				Any	    =					processor(images=__lowerCAmelCase					,       return_tensors="pt"      ).pixel_values

		# original processing pipeline
		_UpperCAmelCase						:				int	    =					Compose(
		    [
		        Resize(__lowerCAmelCase					,       interpolation=pillow_resamplings["bicubic"]      ),
		        CenterCrop(__lowerCAmelCase      ),
		        ToTensor(),
		        Normalize(__lowerCAmelCase					,       __lowerCAmelCase      ),
		    ]      )
		_UpperCAmelCase						:				Any	    =					image_transforms(__lowerCAmelCase      ).unsqueeze(0      )

		assert torch.allclose(__lowerCAmelCase					,       __lowerCAmelCase      )

		_UpperCAmelCase						:				Union[str, Any]	    =					model(__lowerCAmelCase      )
		_UpperCAmelCase						:				Dict	    =					outputs.logits

		_UpperCAmelCase						:				Optional[int]	    =					(1, 1_000)

		if "l1" in model_name:
				_UpperCAmelCase						:				List[Any]	    =					torch.Tensor(
				    [-0.1_3_1_2, 0.4_3_5_3, -1.0_4_9_9, -0.5_1_2_4, 0.4_1_8_3, -0.6_7_9_3, -1.3_7_7_7, -0.0_8_9_3, -0.7_3_5_8, -2.4_3_2_8]      )
				assert torch.allclose(logits[0, :10]					,       __lowerCAmelCase					,       atol=1e-3      )
				assert logits.shape == expected_shape
		elif "l3" in model_name:
				_UpperCAmelCase						:				List[Any]	    =					torch.Tensor(
				    [-1.3_1_5_0, -1.5_4_5_6, -1.2_5_5_6, -0.8_4_9_6, -0.7_1_2_7, -0.7_8_9_7, -0.9_7_2_8, -0.3_0_5_2, 0.3_7_5_1, -0.3_1_2_7]      )
				assert torch.allclose(logits[0, :10]					,       __lowerCAmelCase					,       atol=1e-3      )
				assert logits.shape == expected_shape
		elif "l7" in model_name:
				_UpperCAmelCase						:				List[Any]	    =					torch.Tensor(
				    [-1.0_2_8_3, -1.4_1_3_1, -0.5_6_4_4, -1.3_1_1_5, -0.5_7_8_5, -1.2_0_4_9, -0.7_5_2_8, 0.1_9_9_2, -0.3_8_2_2, -0.0_8_7_8]      )
				assert logits.shape == expected_shape
		else:
				raise ValueError(
				    F"""Unknown model checkpoint: {checkpoint_path}. Supported version of efficientformer are l1, l3 and l7"""      )

		# Save Checkpoints
		Path(__lowerCAmelCase      ).mkdir(exist_ok=__lowerCAmelCase      )
		model.save_pretrained(__lowerCAmelCase      )
		print(F"""Checkpoint successfuly converted. Model saved at {pytorch_dump_path}"""      )
		processor.save_pretrained(__lowerCAmelCase      )
		print(F"""Processor successfuly saved at {pytorch_dump_path}"""      )

		if push_to_hub:
				print("Pushing model to the hub..."      )

				model.push_to_hub(
				    repo_id=F"""Bearnardd/{pytorch_dump_path}"""					,       commit_message="Add model"					,       use_temp_dir=__lowerCAmelCase					,       )
				processor.push_to_hub(
				    repo_id=F"""Bearnardd/{pytorch_dump_path}"""					,       commit_message="Add image processor"					,       use_temp_dir=__lowerCAmelCase					,       )


if __name__ == "__main__":
						lowerCamelCase__					=   argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '--pytorch_model_path',
						    default=None,
						    type=str,
						    required=True,
						    help='Path to EfficientFormer pytorch checkpoint.',
						)
						parser.add_argument(
						    '--config_file',
						    default=None,
						    type=str,
						    required=True,
						    help='The json file for EfficientFormer model config.',
						)
						parser.add_argument(
						    '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
						)

						parser.add_argument('--push_to_hub', action='store_true', help='Push model and image processor to the hub')
						parser.add_argument(
						    '--no-push_to_hub',
						    dest='push_to_hub',
						    action='store_false',
						    help='Do not push model and image processor to the hub',
						)
						parser.set_defaults(push_to_hub=True)

						lowerCamelCase__					=   parser.parse_args()
						convert_efficientformer_checkpoint(
						    checkpoint_path=args.pytorch_model_path,
						    efficientformer_config_file=args.config_file,
						    pytorch_dump_path=args.pytorch_dump_path,
						    push_to_hub=args.push_to_hub,
						)



