Update README.md
Browse files
README.md
CHANGED
@@ -12,6 +12,9 @@ The model of SitEmb-v1.5-Qwen3 trained with additional book notes and their corr
|
|
12 |
import torch
|
13 |
|
14 |
from transformers import AutoTokenizer, AutoModel
|
|
|
|
|
|
|
15 |
|
16 |
residual = True
|
17 |
residual_factor = 0.5
|
@@ -122,8 +125,13 @@ query_hidden, _ = encode_query(
|
|
122 |
tokenizer, model, pooling_type="eos", queries=["Your query"],
|
123 |
batch_size=8, normalize=True, max_length=8192, residual=residual,
|
124 |
)
|
|
|
|
|
|
|
|
|
|
|
125 |
candidate_hidden, candidate_hidden_residual = encode_passage(
|
126 |
-
tokenizer, model, pooling_type="eos", passages=["
|
127 |
batch_size=4, normalize=True, max_length=8192, residual=residual,
|
128 |
)
|
129 |
|
|
|
12 |
import torch
|
13 |
|
14 |
from transformers import AutoTokenizer, AutoModel
|
15 |
+
from tqdm import tqdm
|
16 |
+
from more_itertools import chunked
|
17 |
+
|
18 |
|
19 |
residual = True
|
20 |
residual_factor = 0.5
|
|
|
125 |
tokenizer, model, pooling_type="eos", queries=["Your query"],
|
126 |
batch_size=8, normalize=True, max_length=8192, residual=residual,
|
127 |
)
|
128 |
+
|
129 |
+
passage_affix = "The context in which the chunk is situated is given below. Please encode the chunk by being aware of the context. Context:\n"
|
130 |
+
your_chunk = "Your Chunk"
|
131 |
+
your_context = "Your Context"
|
132 |
+
|
133 |
candidate_hidden, candidate_hidden_residual = encode_passage(
|
134 |
+
tokenizer, model, pooling_type="eos", passages=[f"{your_chunk}<|endoftext|>{passage_affix}{your_context}"],
|
135 |
batch_size=4, normalize=True, max_length=8192, residual=residual,
|
136 |
)
|
137 |
|