



'''simple docstring'''




import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
						from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
						lowerCamelCase							:Tuple				    =					True
						from torch.cuda.amp import autocast


lowerCamelCase							:Optional[int]				    =					logging.getLogger(__name__)







@dataclass
class 	_lowerCAmelCase							:
	__SCREAMING_SNAKE_CASE   :		str					=					field(
	    metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'} )
	__SCREAMING_SNAKE_CASE   :		Optional[str]					=					field(
	    default=__UpperCAmelCase      ,			metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}      ,			)
	__SCREAMING_SNAKE_CASE   :		Optional[bool]					=					field(
	    default=__UpperCAmelCase      ,			metadata={'help': 'Whether to freeze the feature extractor layers of the model.'} )
	__SCREAMING_SNAKE_CASE   :		Optional[bool]					=					field(
	    default=__UpperCAmelCase      ,			metadata={'help': 'Whether to log verbose messages or not.'}      ,			)
	__SCREAMING_SNAKE_CASE   :		Optional[float]					=					field(
	    default=2.0      ,			metadata={'help': 'Maximum temperature for gumbel softmax.'} )
	__SCREAMING_SNAKE_CASE   :		Optional[float]					=					field(
	    default=0.5      ,			metadata={'help': 'Minimum temperature for gumbel softmax.'} )
	__SCREAMING_SNAKE_CASE   :		Optional[float]					=					field(
	    default=0.999_995      ,			metadata={'help': 'Decay of gumbel temperature during training.'} )






def 	a			( lowerCamelCase__						,      lowerCamelCase__    ):
			'''simple docstring'''





			logging.basicConfig(
			    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""						,      datefmt="""%m/%d/%Y %H:%M:%S"""						,      handlers=[logging.StreamHandler(sys.stdout    )]						,      )
			A_	:						Optional[int]	=     logging.WARNING
			if model_args.verbose_logging:
						A_	:						str	=     logging.DEBUG
			elif trainer_utils.is_main_process(training_args.local_rank    ):
						A_	:						Union[str, Any]	=     logging.INFO
			logger.setLevel(lowerCamelCase__    )







@dataclass
class 	_lowerCAmelCase							:
	__SCREAMING_SNAKE_CASE   :		str					=					field(
	    default=__UpperCAmelCase      ,			metadata={'help': 'The name of the dataset to use (via the datasets library).'} )
	__SCREAMING_SNAKE_CASE   :		Optional[str]					=					field(
	    default=__UpperCAmelCase      ,			metadata={'help': 'The configuration name of the dataset to use (via the datasets library).'} )
	__SCREAMING_SNAKE_CASE   :		Optional[str]					=					field(
	    default='train'      ,			metadata={
	        'help': 'The name of the training data set split to use (via the datasets library). Defaults to \'train\''
	    }      ,			)
	__SCREAMING_SNAKE_CASE   :		Optional[str]					=					field(
	    default='validation'      ,			metadata={
	        'help': (
	            'The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''
	        )
	    }      ,			)
	__SCREAMING_SNAKE_CASE   :		Optional[str]					=					field(
	    default='file'      ,			metadata={'help': 'Column in the dataset that contains speech file path. Defaults to \'file\''}      ,			)
	__SCREAMING_SNAKE_CASE   :		bool					=					field(
	    default=__UpperCAmelCase      ,			metadata={'help': 'Overwrite the cached preprocessed datasets or not.'} )
	__SCREAMING_SNAKE_CASE   :		Optional[int]					=					field(
	    default=1      ,			metadata={
	        'help': 'The percentage of the train set used as validation set in case there\'s no validation split'
	    }      ,			)
	__SCREAMING_SNAKE_CASE   :		Optional[int]					=					field(
	    default=__UpperCAmelCase      ,			metadata={'help': 'The number of processes to use for the preprocessing.'}      ,			)
	__SCREAMING_SNAKE_CASE   :		Optional[float]					=					field(
	    default=20.0      ,			metadata={'help': 'Filter audio files that are longer than `max_duration_in_seconds` seconds'} )







@dataclass
class 	_lowerCAmelCase							:
	__SCREAMING_SNAKE_CASE   :		WavaVecaForPreTraining
	__SCREAMING_SNAKE_CASE   :		WavaVecaFeatureExtractor
	__SCREAMING_SNAKE_CASE   :		Union[bool, str]					=					"longest"
	__SCREAMING_SNAKE_CASE   :		Optional[int]					=					None
	__SCREAMING_SNAKE_CASE   :		Optional[int]					=					None


	def __call__(self       ,      lowercase							):
				# reformat list to dict and set to pytorch format
				A_	:						Optional[int]	=     self.feature_extractor.pad(
				    lowercase       ,      max_length=self.max_length       ,      padding=self.padding       ,      pad_to_multiple_of=self.pad_to_multiple_of       ,      return_tensors="""pt"""       ,      )
				A_	:						Union[str, Any]	=     self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1]							)

				A_	:						int	=     batch["""input_values"""].shape[0]

				# make sure that no loss is computed on padded inputs
				if batch["attention_mask"] is not None:
							# compute real output lengths according to convolution formula
							A_	:						Optional[int]	=     self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1							)							).to(
							    torch.long							)

							A_	:						Union[str, Any]	=     torch.zeros(
							    (batch_size, mask_indices_seq_length)       ,      dtype=torch.long       ,      device=batch["""input_values"""].device							)

							# these two operations makes sure that all values
							# before the output lengths indices are attended to
							A_	:						Union[str, Any]	=     1
							A_	:						Tuple	=     attention_mask.flip([-1]							).cumsum(-1							).flip([-1]							).bool()

				# sample randomly masked indices
				A_	:						str	=     _compute_mask_indices(
				    (batch_size, mask_indices_seq_length)       ,      self.model.config.mask_time_prob       ,      self.model.config.mask_time_length       ,      attention_mask=lowercase       ,      min_masks=2       ,      )

				return batch







class 	_lowerCAmelCase							(			__UpperCAmelCase ):


	def __init__(self       ,      *lowercase       ,      lowercase=1       ,      lowercase=0       ,      lowercase=1.0       ,      **lowercase							):
				super().__init__(*lowercase       ,      **lowercase							)
				A_	:						Optional[Any]	=     0
				A_	:						Dict	=     max_gumbel_temp
				A_	:						int	=     min_gumbel_temp
				A_	:						List[str]	=     gumbel_temp_decay


	def     _a	(self       ,      lowercase       ,      lowercase							):
				model.train()
				A_	:						Tuple	=     self._prepare_inputs(lowercase							)

				if self.use_amp:
							with autocast():
										A_	:						List[str]	=     self.compute_loss(lowercase       ,      lowercase							)
				else:
							A_	:						Union[str, Any]	=     self.compute_loss(lowercase       ,      lowercase							)

				if self.args.n_gpu > 1 or self.deepspeed:
							if model.module.config.ctc_loss_reduction == "mean":
										A_	:						Union[str, Any]	=     loss.mean()
							elif model.module.config.ctc_loss_reduction == "sum":
										A_	:						List[Any]	=     loss.sum() / (inputs["""mask_time_indices"""]).sum()
							else:
										raise ValueError(F'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'							)

				if self.args.gradient_accumulation_steps > 1:
							A_	:						List[Any]	=     loss / self.args.gradient_accumulation_steps

				if self.use_amp:
							self.scaler.scale(lowercase							).backward()
				elif self.use_apex:
							with amp.scale_loss(lowercase       ,      self.optimizer							) as scaled_loss:
										scaled_loss.backward()
				elif self.deepspeed:
							self.deepspeed.backward(lowercase							)
				else:
							loss.backward()

				self.num_update_step += 1
				# make sure gumbel softmax temperature is decayed
				if self.args.n_gpu > 1 or self.deepspeed:
							model.module.set_gumbel_temperature(
							    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       ,      self.min_gumbel_temp							)							)
				else:
							model.set_gumbel_temperature(
							    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       ,      self.min_gumbel_temp							)							)

				return loss.detach()






def 	a			( ):
			'''simple docstring'''





			A_	:						Tuple	=     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)    )

			A_,    A_,    A_	:						Dict	=     parser.parse_args_into_dataclasses()
			configure_logger(lowerCamelCase__						,      lowerCamelCase__    )

			# Downloading and loading a dataset from the hub.
			A_	:						Optional[int]	=     load_dataset(data_args.dataset_name						,      data_args.dataset_config_name						,      cache_dir=model_args.cache_dir    )

			if "validation" not in datasets.keys():
						# make sure only "validation" and "train" keys remain"
						A_	:						str	=     DatasetDict()
						A_	:						List[str]	=     load_dataset(
						    data_args.dataset_name						,      data_args.dataset_config_name						,      split=f'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'						,      cache_dir=model_args.cache_dir						,      )
						A_	:						Tuple	=     load_dataset(
						    data_args.dataset_name						,      data_args.dataset_config_name						,      split=f'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'						,      cache_dir=model_args.cache_dir						,      )
			else:
						# make sure only "validation" and "train" keys remain"
						A_	:						Tuple	=     DatasetDict()
						A_	:						int	=     load_dataset(
						    data_args.dataset_name						,      data_args.dataset_config_name						,      split="""validation"""						,      cache_dir=model_args.cache_dir						,      )
						A_	:						Any	=     load_dataset(
						    data_args.dataset_name						,      data_args.dataset_config_name						,      split=f'{data_args.train_split_name}'						,      cache_dir=model_args.cache_dir						,      )

			# only normalized-inputs-training is supported
			A_	:						str	=     WavaVecaFeatureExtractor.from_pretrained(
			    model_args.model_name_or_path						,      cache_dir=model_args.cache_dir						,      do_normalize=lowerCamelCase__    )

			def prepare_dataset(lowerCamelCase__    ):
						# check that all files have the correct sampling rate
						A_,    A_	:						Dict	=     librosa.load(batch[data_args.speech_file_column]						,      sr=feature_extractor.sampling_rate    )
						return batch

			# load audio files into numpy arrays
			A_	:						Dict	=     datasets.map(
			    lowerCamelCase__						,      num_proc=data_args.preprocessing_num_workers						,      remove_columns=datasets["""train"""].column_names    )

			# filter audio files that are too long
			A_	:						Optional[int]	=     vectorized_datasets.filter(
			    lambda lowerCamelCase__    : len(data["""speech"""]    ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate    )    )

			def normalize(lowerCamelCase__    ):
						return feature_extractor(batch["""speech"""]						,      sampling_rate=feature_extractor.sampling_rate    )

			# normalize and transform to `BatchFeatures`
			A_	:						Any	=     vectorized_datasets.map(
			    lowerCamelCase__						,      batched=lowerCamelCase__						,      num_proc=data_args.preprocessing_num_workers						,      load_from_cache_file=not data_args.overwrite_cache						,      remove_columns=vectorized_datasets["""train"""].column_names						,      )

			# pretraining is only supported for "newer" stable layer norm architecture
			# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
			A_	:						List[Any]	=     WavaVecaConfig.from_pretrained(
			    model_args.model_name_or_path						,      cache_dir=model_args.cache_dir						,      gradient_checkpointing=training_args.gradient_checkpointing						,      )

			if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
						raise ValueError(
						    """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
						    """ ``config.feat_extract_norm='layer'"""    )

			A_	:						Optional[Any]	=     WavaVecaForPreTraining(lowerCamelCase__    )

			A_	:						Dict	=     DataCollatorForWavaVecaPretraining(model=lowerCamelCase__						,      feature_extractor=lowerCamelCase__    )

			A_	:						Optional[int]	=     WavaVecaPreTrainer(
			    model=lowerCamelCase__						,      data_collator=lowerCamelCase__						,      args=lowerCamelCase__						,      train_dataset=vectorized_datasets["""train"""]						,      eval_dataset=vectorized_datasets["""validation"""]						,      tokenizer=lowerCamelCase__						,      max_gumbel_temp=model_args.max_gumbel_temperature						,      min_gumbel_temp=model_args.min_gumbel_temperature						,      gumbel_temp_decay=model_args.gumbel_temperature_decay						,      )
			trainer.train()


if __name__ == "__main__":
						main()




'''simple docstring'''




# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from .config import config_command_parser
from .config_args import default_config_file, load_config_from_file  # noqa: F401
from .default import default_command_parser
from .update import update_command_parser



def 	a			( lowerCamelCase__=None    ):
			'''simple docstring'''





			A_	:						int	=     argparse.ArgumentParser(add_help=lowerCamelCase__						,      allow_abbrev=lowerCamelCase__    )
			# The main config parser
			A_	:						int	=     config_command_parser(lowerCamelCase__    )
			# The subparser to add commands to
			A_	:						int	=     config_parser.add_subparsers(title="""subcommands"""						,      dest="""subcommand"""    )

			# Then add other parsers with the parent parser
			default_command_parser(lowerCamelCase__						,      parents=[parent_parser]    )
			update_command_parser(lowerCamelCase__						,      parents=[parent_parser]    )

			return config_parser



def 	a			( ):
			'''simple docstring'''





			A_	:						Optional[int]	=     get_config_parser()
			A_	:						List[str]	=     config_parser.parse_args()

			if not hasattr(lowerCamelCase__						,      """func"""    ):
						config_parser.print_help()
						exit(1    )

			# Run
			args.func(lowerCamelCase__    )


if __name__ == "__main__":
						main()