pytorch
/

Qwen3-8B-INT4

Text Generation

text-generation-inference

Model card Files Files and versions

jerryzh168 commited on 5 days ago

Commit

45cf0e8

·

verified ·

1 Parent(s): e6ebd63

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -125,7 +125,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, TorchAoConfig
 model_id = "Qwen/Qwen3-8B"
 from torchao.quantization import Int4WeightOnlyConfig
-quant_config = Int4WeightOnlyConfig(group_size=128, use_hqq=True)
 quantization_config = TorchAoConfig(quant_type=quant_config)
 quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
 tokenizer = AutoTokenizer.from_pretrained(model_id)

 model_id = "Qwen/Qwen3-8B"
 from torchao.quantization import Int4WeightOnlyConfig
+quant_config = Int4WeightOnlyConfig(group_size=128, int4_packing_format="tile_packed_to_4d", int4_choose_qparams_algorithm="hqq")
 quantization_config = TorchAoConfig(quant_type=quant_config)
 quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
 tokenizer = AutoTokenizer.from_pretrained(model_id)