Update README.md
Browse files
README.md
CHANGED
|
@@ -42,7 +42,7 @@ max_model_len, tp_size = 4096, 1
|
|
| 42 |
model_name = "neuralmagic-ent/DeepSeek-R1-Distill-Llama-8B-FP8-Dynamic"
|
| 43 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 44 |
llm = LLM(model=model_name, tensor_parallel_size=tp_size, max_model_len=max_model_len, trust_remote_code=True)
|
| 45 |
-
sampling_params = SamplingParams(temperature=0.
|
| 46 |
|
| 47 |
messages_list = [
|
| 48 |
[{"role": "user", "content": "Who are you? Please respond in pirate speak!"}],
|
|
|
|
| 42 |
model_name = "neuralmagic-ent/DeepSeek-R1-Distill-Llama-8B-FP8-Dynamic"
|
| 43 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 44 |
llm = LLM(model=model_name, tensor_parallel_size=tp_size, max_model_len=max_model_len, trust_remote_code=True)
|
| 45 |
+
sampling_params = SamplingParams(temperature=0.6, max_tokens=256, stop_token_ids=[tokenizer.eos_token_id])
|
| 46 |
|
| 47 |
messages_list = [
|
| 48 |
[{"role": "user", "content": "Who are you? Please respond in pirate speak!"}],
|