Update README.md

README.md

@@ -45,3 +45,147 @@ pip install transformers==4.45.2
 
 ### 4.  Inference
 The following script loads the model and allows inference.
+
+```python
+from llava.model.builder import load_pretrained_model
+from llava.mm_utils import get_model_name_from_path, process_images, tokenizer_image_token
+from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX
+from llava.conversation import conv_templates, SeparatorStyle
+
+from PIL import Image
+import copy
+import torch
+
+import warnings
+warnings.filterwarnings("ignore")
+
+
+pretrained = 'r-g2-2024/Llama-3.1-70B-Instruct-multimodal-JP-Graph-v0.1'
+model_name = "llava_llama"
+device = "cuda"
+device_map = "auto"
+tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map=device_map)
+
+model.eval()
+
+image = Image.open("./画像14.png")
+image
+
+inputs = image_processor(image)
+pixel_values = torch.tensor(inputs['pixel_values']).to(dtype=torch.float16, device=device)
+pixel_values = [pixel_values]
+_image_grid_thw = torch.tensor(inputs['image_grid_thw'], dtype=torch.long)
+_image_grid_thw = [_image_grid_thw]
+
+conv_template = "llava_llama_3"
+question = DEFAULT_IMAGE_TOKEN + "\nFY22からFY23にかけて単体の値はどれくらい増加したか？"
+conv = copy.deepcopy(conv_templates[conv_template])
+conv.append_message(conv.roles[0], question)
+conv.append_message(conv.roles[1], None)
+prompt_question = conv.get_prompt()
+
+input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(device)
+image_sizes = [image.size]
+
+cont = model.generate(
+    input_ids,
+    images=pixel_values,
+    image_sizes=image_sizes,
+    image_grid_thws=_image_grid_thw,
+    do_sample=False,
+    temperature=0,
+    max_new_tokens=4096,
+)
+text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
+print(text_outputs)
+
+
+
+question = DEFAULT_IMAGE_TOKEN + "\nFY2021の連結の値はいくつか？"
+conv = copy.deepcopy(conv_templates[conv_template])
+conv.append_message(conv.roles[0], question)
+conv.append_message(conv.roles[1], None)
+prompt_question = conv.get_prompt()
+
+input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(device)
+image_sizes = [image.size]
+
+cont = model.generate(
+    input_ids,
+    images=pixel_values,
+    image_sizes=image_sizes,
+    image_grid_thws=_image_grid_thw,
+    do_sample=False,
+    temperature=0,
+    max_new_tokens=4096,
+)
+text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
+print(text_outputs)
+
+question = DEFAULT_IMAGE_TOKEN + "\nこの図は何を表しているか？"
+conv = copy.deepcopy(conv_templates[conv_template])
+conv.append_message(conv.roles[0], question)
+conv.append_message(conv.roles[1], None)
+prompt_question = conv.get_prompt()
+
+input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(device)
+image_sizes = [image.size]
+
+cont = model.generate(
+    input_ids,
+    images=pixel_values,
+    image_sizes=image_sizes,
+    image_grid_thws=_image_grid_thw,
+    do_sample=False,
+    temperature=0,
+    max_new_tokens=4096,
+)
+text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
+print(text_outputs)
+
+
+question = DEFAULT_IMAGE_TOKEN + "\nFY2020の純利益はマイナスか？プラスか？"
+conv = copy.deepcopy(conv_templates[conv_template])
+conv.append_message(conv.roles[0], question)
+conv.append_message(conv.roles[1], None)
+prompt_question = conv.get_prompt()
+
+input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(device)
+image_sizes = [image.size]
+
+cont = model.generate(
+    input_ids,
+    images=pixel_values,
+    image_sizes=image_sizes,
+    image_grid_thws=_image_grid_thw,
+    do_sample=False,
+    temperature=0,
+    max_new_tokens=4096,
+)
+text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
+print(text_outputs)
+
+
+question = DEFAULT_IMAGE_TOKEN + "\n単体が連結の利益を上回るのはいつからか？"
+conv = copy.deepcopy(conv_templates[conv_template])
+conv.append_message(conv.roles[0], question)
+conv.append_message(conv.roles[1], None)
+prompt_question = conv.get_prompt()
+
+input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(device)
+image_sizes = [image.size]
+
+cont = model.generate(
+    input_ids,
+    images=pixel_values,
+    image_sizes=image_sizes,
+    image_grid_thws=_image_grid_thw,
+    do_sample=False,
+    temperature=0,
+    max_new_tokens=4096,
+)
+text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
+print(text_outputs)
+
+
+```