Fine-tuned openai/whisper-small on multilingual dataset

README.md

@@ -4,31 +4,43 @@ language:
 - ms
 - zh
 - en
-license: mit
-base_model: openai/whisper-large-v3-turbo
+license: apache-2.0
+base_model: openai/whisper-small
 tags:
+- whisper
+- multilingual
+- speech-recognition
 - generated_from_trainer
 datasets:
 - CheeseES/LLM_FINE_TUNING_1
+metrics:
+- wer
 model-index:
-- name: Whisper_Large_V3_Turbo_Tune
-  results: []
+- name: Whisper_FT_V1
+  results:
+  - task:
+      type: automatic-speech-recognition
+      name: Automatic Speech Recognition
+    dataset:
+      name: LLM Fine Tuning Dataset
+      type: CheeseES/LLM_FINE_TUNING_1
+      split: None
+      args: language
+    metrics:
+    - type: wer
+      value: 51.61904761904762
+      name: Wer
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# Whisper_Large_V3_Turbo_Tune
+# Whisper_FT_V1
 
-This model is a fine-tuned version of [openai/whisper-large-v3-turbo](https://huggingface.co/openai/whisper-large-v3-turbo) on the LLM Fine Tuning Dataset dataset.
+This model is a fine-tuned version of [openai/whisper-small](https://huggingface.co/openai/whisper-small) on the LLM Fine Tuning Dataset dataset.
 It achieves the following results on the evaluation set:
-- eval_loss: 0.0558
-- eval_wer: 48.2143
-- eval_runtime: 216.6063
-- eval_samples_per_second: 1.08
-- eval_steps_per_second: 0.139
-- epoch: 10.2564
-- step: 1200
+- Loss: 0.0892
+- Wer: 51.6190
 
 ## Model description
 
@@ -52,11 +64,35 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 33
 - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
-- lr_scheduler_type: cosine
+- lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 300
 - training_steps: 3000
 - mixed_precision_training: Native AMP
 
+### Training results
+
+| Training Loss | Epoch   | Step | Validation Loss | Wer     |
+|:-------------:|:-------:|:----:|:---------------:|:-------:|
+| 0.5149        | 0.8547  | 100  | 0.2080          | 80.4524 |
+| 0.2           | 1.7094  | 200  | 0.1883          | 82.9286 |
+| 0.1807        | 2.5641  | 300  | 0.1701          | 84.7143 |
+| 0.1561        | 3.4188  | 400  | 0.1553          | 82.6667 |
+| 0.1363        | 4.2735  | 500  | 0.1458          | 75.3571 |
+| 0.1152        | 5.1282  | 600  | 0.1367          | 71.3095 |
+| 0.0994        | 5.9829  | 700  | 0.1284          | 68.6190 |
+| 0.0865        | 6.8376  | 800  | 0.1214          | 64.5238 |
+| 0.073         | 7.6923  | 900  | 0.1136          | 69.5714 |
+| 0.0656        | 8.5470  | 1000 | 0.1091          | 66.6905 |
+| 0.0598        | 9.4017  | 1100 | 0.1049          | 69.8810 |
+| 0.0512        | 10.2564 | 1200 | 0.1025          | 65.0    |
+| 0.0481        | 11.1111 | 1300 | 0.0977          | 64.8571 |
+| 0.0429        | 11.9658 | 1400 | 0.0955          | 59.5238 |
+| 0.0385        | 12.8205 | 1500 | 0.0930          | 61.3810 |
+| 0.0338        | 13.6752 | 1600 | 0.0916          | 65.3810 |
+| 0.0334        | 14.5299 | 1700 | 0.0905          | 63.0952 |
+| 0.0298        | 15.3846 | 1800 | 0.0892          | 51.6190 |
+
+
 ### Framework versions
 
 - PEFT 0.15.2

adapter_config.json

@@ -4,7 +4,7 @@
     "base_model_class": "WhisperForConditionalGeneration",
     "parent_library": "transformers.models.whisper.modeling_whisper"
   },
-  "base_model_name_or_path": "openai/whisper-large-v3-turbo",
+  "base_model_name_or_path": "openai/whisper-small",
   "bias": "none",
   "corda_config": null,
   "eva_config": null,
@@ -27,12 +27,12 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "fc1",
-    "q_proj",
     "fc2",
     "out_proj",
     "v_proj",
-    "k_proj"
+    "k_proj",
+    "fc1",
+    "q_proj"
   ],
   "task_type": null,
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd18fdec9de19ed00700f9f8f913bc329f56cc8f6a4eb806c88b96bef3fe9aad
-size 27916528
+oid sha256:df5f188bcadd61ac856942ff0342f2c605ff5feae5ff4d49616acb054f07d7ab
+size 13028552

preprocessor_config.json

@@ -2,7 +2,7 @@
   "chunk_length": 30,
   "dither": 0.0,
   "feature_extractor_type": "WhisperFeatureExtractor",
-  "feature_size": 128,
+  "feature_size": 80,
   "hop_length": 160,
   "n_fft": 400,
   "n_samples": 480000,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43d1137e7a3b20a887997470a211a01efcce3749d35275a32598a5ab41a125d4
-size 5841
+oid sha256:0269ac6e49a474c3f8b6ccc7ef18792747d434a732d842fcb264e60e7bcf2aee
+size 8081