Update README.md

README.md

@@ -108,6 +108,7 @@ vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://do
   from llmcompressor.modifiers.quantization import GPTQModifier
   from llmcompressor.transformers import oneshot
   from llmcompressor.transformers.tracing import TraceableMistral3ForConditionalGeneration
+  from datasets import load_dataset, interleave_datasets
   from PIL import Image
   import io
   
@@ -116,7 +117,7 @@ vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://do
   model_name = model_stub.split("/")[-1]
   
   num_text_samples = 1024
-  num_vison_samples = 1024
+  num_vision_samples = 1024
   max_seq_len = 8192
   
   processor = AutoProcessor.from_pretrained(model_stub)
@@ -134,13 +135,14 @@ vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://do
               example["messages"],
               add_generation_prompt=False,
           ),
-          "images" = None,
+          "images": None,
       }
       tokenized_input = processor(**input, max_length=max_seq_len, truncation=True)
       tokenized_input["pixel_values"] = tokenized_input.get("pixel_values", None)
       tokenized_input["image_sizes"] = tokenized_input.get("image_sizes", None)
+      return tokenized_input
 
-  dst = load_dataset("neuralmagic/calibration", name="LLM", split="train").select(num_text_samples)
+  dst = load_dataset("neuralmagic/calibration", name="LLM", split="train").select(range(num_text_samples))
   dst = dst.map(preprocess_text, remove_columns=dst.column_names)
 
   # Text + vision data subset
@@ -149,11 +151,11 @@ vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://do
       image = None
       for message in example["messages"]:
           message_content = []
-          for content in message["content"]
+          for content in message["content"]:
               if content["type"] == "text":
-                  message_content = {"type": "text", "text": content["text"]}
+                  message_content.append({"type": "text", "text": content["text"]})
               else:
-                  message_content = {"type": "image"}}
+                  message_content.append({"type": "image"})
                   image = Image.open(io.BytesIO(content["image"]))
 
           messages.append(
@@ -168,13 +170,14 @@ vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://do
               messages,
               add_generation_prompt=False,
           ),
-          "images" = image,
+          "images": image,
       }
       tokenized_input = processor(**input, max_length=max_seq_len, truncation=True)
       tokenized_input["pixel_values"] = tokenized_input.get("pixel_values", None)
       tokenized_input["image_sizes"] = tokenized_input.get("image_sizes", None)
+      return tokenized_input
 
-  dsv = load_dataset("neuralmagic/calibration", name="VLLM", split="train").select(num_vision_samples)
+  dsv = load_dataset("neuralmagic/calibration", name="VLM", split="train").select(range(num_vision_samples))
   dsv = dsv.map(preprocess_vision, remove_columns=dsv.column_names)
 
   # Interleave subsets
@@ -182,9 +185,9 @@ vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://do
 
   # Configure the quantization algorithm and scheme
   recipe = GPTQModifier(
-      ignore=["language_model.lm_head", "re:vision_tower.*", "re:multi_modal_projector.*"]
-      sequential_targets=["MistralDecoderLayer"]
-      dampening_frac=0.01
+      ignore=["language_model.lm_head", "re:vision_tower.*", "re:multi_modal_projector.*"],
+      sequential_targets=["MistralDecoderLayer"],
+      dampening_frac=0.01,
       targets="Linear",
       scheme="W4A16",
   )
@@ -213,12 +216,13 @@ vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://do
       recipe=recipe,
       max_seq_length=max_seq_len,
       data_collator=data_collator,
+      num_samples=num_text_samples + num_vision_samples,
   )
   
   # Save to disk in compressed-tensors format
-  save_path = model_name + "-quantized.w4a16
+  save_path = model_name + "-quantized.w4a16"
   model.save_pretrained(save_path)
-  tokenizer.save_pretrained(save_path)
+  processor.save_pretrained(save_path)
   print(f"Model and tokenizer saved to: {save_path}")
   ```
 </details>