Update README.md

README.md

@@ -3691,57 +3691,94 @@ The `GME` models support three types of input: **text**, **image**, and **image-
 |[`gme-Qwen2-VL-2B`](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct) | 2.21B | 32768 | 1536 | 65.27 | 68.41 | 64.45 |
 |[`gme-Qwen2-VL-7B`](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct) | 8.29B | 32768 | 3584 | 67.48 | 71.36 | 67.44 |
 
+
+
 ## Usage 
-**Use with custom code**
 
-```python
-# You can find the script gme_inference.py in https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct/blob/main/gme_inference.py
-from gme_inference import GmeQwen2VL
 
-model = GmeQwen2VL('Alibaba-NLP/gme-Qwen2-VL-7B-Instruct')
+**Transformers**
 
+```python
+t2i_prompt = 'Find an image that matches the given text.'
 texts = [
-    "What kind of car is this?",
-    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023."
+    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023.",
+    "Alibaba office.",
 ]
 images = [
-    'https://en.wikipedia.org/wiki/File:Tesla_Cybertruck_damaged_window.jpg',
-    'https://en.wikipedia.org/wiki/File:2024_Tesla_Cybertruck_Foundation_Series,_front_left_(Greenwich).jpg',
+    'https://upload.wikimedia.org/wikipedia/commons/e/e9/Tesla_Cybertruck_damaged_window.jpg',
+    'https://upload.wikimedia.org/wikipedia/commons/e/e0/TaobaoCity_Alibaba_Xixi_Park.jpg',
 ]
 
+
+gme = AutoModel.from_pretrained(
+    "Alibaba-NLP/gme-Qwen2-VL-7B-Instruct",
+    torch_dtype="float16", device_map='cuda', trust_remote_code=True
+)
+
+
 # Single-modal embedding
 e_text = gme.get_text_embeddings(texts=texts)
 e_image = gme.get_image_embeddings(images=images)
-print((e_text * e_image).sum(-1))
-## tensor([0.1702, 0.5278], dtype=torch.float16)
+print('Single-modal', (e_text @ e_image.T).tolist())
+## Single-modal [[0.279296875, 0.0002658367156982422], [0.06427001953125, 0.304443359375]]
 
 # How to set embedding instruction
-e_query = gme.get_text_embeddings(texts=texts, instruction='Find an image that matches the given text.')
+e_query = gme.get_text_embeddings(texts=texts, instruction=t2i_prompt)
 # If is_query=False, we always use the default instruction.
 e_corpus = gme.get_image_embeddings(images=images, is_query=False)
-print((e_query * e_corpus).sum(-1))
-## tensor([0.2000, 0.5752], dtype=torch.float16)
+print('Single-modal with instruction', (e_query @ e_corpus.T).tolist())
+## Single-modal with instruction [[0.32861328125, 0.026336669921875], [0.09466552734375, 0.3134765625]]
 
 # Fused-modal embedding
 e_fused = gme.get_fused_embeddings(texts=texts, images=images)
-print((e_fused[0] * e_fused[1]).sum())
-## tensor(0.6826, dtype=torch.float16)
-
+print('Fused-modal', (e_fused @ e_fused.T).tolist())
+## Fused-modal [[1.0, 0.0308685302734375], [0.0308685302734375, 1.0]]
 ```
 
-<!-- <details>
-<summary>With transformers</summary>
+
+**sentence_transformers**
+
+The `encode` function accept `str` or `dict` with key(s) in `{'text', 'image', 'prompt'}`.
+
+**Do not pass `prompt` as the argument to `encode`**, pass as the input as a `dict` with a `prompt` key.
 
 ```python
-# Requires transformers>=4.46.2
+from sentence_transformers import SentenceTransformer
+
 
-TODO
+t2i_prompt = 'Find an image that matches the given text.'
+texts = [
+    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023.",
+    "Alibaba office.",
+]
+images = [
+    'https://upload.wikimedia.org/wikipedia/commons/e/e9/Tesla_Cybertruck_damaged_window.jpg',
+    'https://upload.wikimedia.org/wikipedia/commons/e/e0/TaobaoCity_Alibaba_Xixi_Park.jpg',
+]
+
+
+gme_st = SentenceTransformer("Alibaba-NLP/gme-Qwen2-VL-7B-Instruct")
 
-# [[0.3016996383666992, 0.7503870129585266, 0.3203084468841553]]
+# Single-modal embedding
+e_text = gme_st.encode(texts, convert_to_tensor=True)
+e_image = gme_st.encode([dict(image=i) for i in images], convert_to_tensor=True)
+print('Single-modal', (e_text @ e_image.T).tolist())
+## Single-modal [[0.27880859375, 0.0005745887756347656], [0.06500244140625, 0.306640625]]
+
+# How to set embedding instruction
+e_query = gme_st.encode([dict(text=t, prompt=t2i_prompt) for t in texts], convert_to_tensor=True)
+# If no prompt, we always use the default instruction.
+e_corpus = gme_st.encode([dict(image=i) for i in images], convert_to_tensor=True)
+print('Single-modal with instruction', (e_query @ e_corpus.T).tolist())
+## Single-modal with instruction [[0.328369140625, 0.0269927978515625], [0.09521484375, 0.316162109375]]
+
+# Fused-modal embedding
+e_fused = gme_st.encode([dict(text=t, image=i) for t, i in zip(texts, images)], convert_to_tensor=True)
+print('Fused-modal', (e_fused @ e_fused.T).tolist())
+## Fused-modal [[0.99951171875, 0.0311737060546875], [0.0311737060546875, 1.0009765625]]
 ```
 
-</details>
- -->
+
 
 ## Evaluation