Update README.md
Browse files
README.md
CHANGED
|
@@ -59,8 +59,8 @@ if __name__ == '__main__':
|
|
| 59 |
tokenizer = AutoTokenizer.from_pretrained("inclusionAI/Ring-mini-linear-2.0-GPTQ-int4")
|
| 60 |
|
| 61 |
sampling_params = SamplingParams(temperature=0.6, top_p=1.0, max_tokens=16384)
|
| 62 |
-
|
| 63 |
-
|
| 64 |
llm = LLM(model="inclusionAI/Ring-mini-linear-2.0-GPTQ-int4", dtype='auto', enable_prefix_caching=False, max_num_seqs=128)
|
| 65 |
|
| 66 |
|
|
@@ -76,7 +76,7 @@ if __name__ == '__main__':
|
|
| 76 |
)
|
| 77 |
outputs = llm.generate([text], sampling_params)
|
| 78 |
for output in outputs:
|
| 79 |
-
print(output.outputs.)
|
| 80 |
```
|
| 81 |
|
| 82 |
#### Online Inference
|
|
|
|
| 59 |
tokenizer = AutoTokenizer.from_pretrained("inclusionAI/Ring-mini-linear-2.0-GPTQ-int4")
|
| 60 |
|
| 61 |
sampling_params = SamplingParams(temperature=0.6, top_p=1.0, max_tokens=16384)
|
| 62 |
+
|
| 63 |
+
# use `max_num_seqs=1` without concurrency
|
| 64 |
llm = LLM(model="inclusionAI/Ring-mini-linear-2.0-GPTQ-int4", dtype='auto', enable_prefix_caching=False, max_num_seqs=128)
|
| 65 |
|
| 66 |
|
|
|
|
| 76 |
)
|
| 77 |
outputs = llm.generate([text], sampling_params)
|
| 78 |
for output in outputs:
|
| 79 |
+
print(output.outputs[0].text)
|
| 80 |
```
|
| 81 |
|
| 82 |
#### Online Inference
|