Create benchmark.py

benchmark.py

@@ -0,0 +1,27 @@
+from pprint import pprint
+from transformers import pipeline
+from datasets import load_dataset
+
+# config
+model_id = "kotoba-tech/kotoba-whisper-v1.0"
+generate_kwargs = {"language": "japanese", "task": "transcribe"}
+
+# load model
+pipe = pipeline(
+    "automatic-speech-recognition",
+    model=model_id,
+    chunk_length_s=15,
+    batch_size=64
+)
+test_audio = [
+    "kotoba-whisper-eval/audio/long_interview_1.wav",
+    "kotoba-whisper-eval/audio/manzai1.wav",
+    "kotoba-whisper-eval/audio/manzai2.wav",
+    "kotoba-whisper-eval/audio/manzai3.wav"
+]
+elapsed = {}
+for x in test_audio:
+    start = time()
+    transcription = pipe(x, generate_kwargs=generate_kwargs)
+    elapsed[x] = time() - start
+pprint(elapsed)