多模态使用方法

api

代码实现

@RequestMapping("/multiModal")
@RestController
public class MultiModalController {
    @Autowired
    ChatModel chatModel;

    @GetMapping()
    public String multiModal(@RequestParam(value = "msg",
                                  defaultValue = "你从这张图中看到了什么") String msg) throws IOException {
        byte[] imageData = new ClassPathResource("static/images/test.png").getContentAsByteArray();

        UserMessage userMessage = new UserMessage(msg, List.of(new Media(MimeTypeUtils.IMAGE_PNG,imageData)));
        ChatResponse response = chatModel.call(new Prompt(userMessage,
                OpenAiChatOptions.builder()
                        .withModel(OpenAiApi.ChatModel.GPT_4_TURBO_PREVIEW.getValue())
                        .build()));
        return response.getResult().getOutput().getContent();
    }
}

图片理解