Update handler.py

handler.py

@@ -125,6 +125,7 @@ class EndpointHandler:
                     trust_remote_code=True,
                     device_map=device_map,
                     torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
+                    quantization_config=None,  # Disable model's built-in quantization
                     low_cpu_mem_usage=True,
                     offload_folder=offload_folder if device_map == "auto" else None,
                     max_memory=max_memory,
@@ -144,6 +145,7 @@ class EndpointHandler:
                     model_path,
                     trust_remote_code=True,
                     torch_dtype=torch.float32,
+                    quantization_config=None,  # Disable model's built-in quantization
                     low_cpu_mem_usage=True,
                 )
                 logger.info("Successfully loaded with FP32 on CPU")