Upload UltravoxPipeline

config.json

@@ -154,16 +154,10 @@
   },
   "custom_pipelines": {
     "ultravox-pipeline": {
-      "default": {
-        "model": {
-          "pt": [
-            "fixie-ai/ultravox-v0.2",
-            "main"
-          ]
-        }
-      },
       "impl": "ultravox_pipeline.UltravoxPipeline",
-      "pt": ["AutoModel"],
+      "pt": [
+        "AutoModel"
+      ],
       "tf": [],
       "type": "multimodal"
     }
@@ -193,6 +187,6 @@
   },
   "text_model_id": null,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.2",
+  "transformers_version": "4.41.1",
   "vocab_size": 128256
 }

generation_config.json

@@ -2,5 +2,6 @@
   "_from_model_config": true,
   "bos_token_id": 128000,
   "eos_token_id": 128009,
-  "transformers_version": "4.41.2"
+  "pad_token_id": 128009,
+  "transformers_version": "4.41.1"
 }

special_tokens_map.json

@@ -13,5 +13,11 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|eot_id|>"
+  "pad_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

ultravox_pipeline.py

@@ -67,11 +67,15 @@ class UltravoxPipeline(transformers.Pipeline):
                 "No sampling rate provided, using default of 16kHz. We highly recommend providing the correct sampling rate."
             )
 
-        return self.processor(
+        output = self.processor(
             text=text,
             audio=inputs["audio"],
             sampling_rate=inputs.get("sampling_rate", 16000),
         )
+        if "audio_values" in output:
+            output["audio_values"] = output["audio_values"].to(self.model.dtype)
+
+        return output
 
     def _forward(
         self,