xieyinlu/ruoyi-ai
1
0
Fork 0
mirror of https://gitcode.com/ageerle/ruoyi-ai.git synced 2026-03-14 05:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 图片识别功能优化

Browse Source
This commit is contained in:
ageerle
2025-06-05 16:00:06 +08:00
parent 353fbf26b8
commit baa664ac4f
5 changed files with 161 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java
View File

@@ -1,140 +0,0 @@
package org.ruoyi.chat.service.chat.impl;
import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversation;
import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationParam;
import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationResult;
import com.alibaba.dashscope.common.MultiModalMessage;
import com.alibaba.dashscope.common.ResultCallback;
import com.alibaba.dashscope.common.Role;
import com.alibaba.dashscope.exception.NoApiKeyException;
import com.alibaba.dashscope.exception.UploadFileException;
import dev.langchain4j.community.model.dashscope.QwenStreamingChatModel;
import dev.langchain4j.community.model.dashscope.QwenStreamingResponseBuilder;
import dev.langchain4j.data.message.ChatMessage;
import dev.langchain4j.data.message.SystemMessage;
import dev.langchain4j.data.message.UserMessage;
import dev.langchain4j.internal.Utils;
import dev.langchain4j.model.chat.StreamingChatModel;
import dev.langchain4j.model.chat.response.ChatResponse;
import dev.langchain4j.model.chat.response.StreamingChatResponseHandler;
import io.modelcontextprotocol.client.McpSyncClient;
import io.reactivex.Flowable;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.chat.config.ChatConfig;
import org.ruoyi.chat.enums.ChatModeType;
import org.ruoyi.chat.listener.SSEEventSourceListener;
import org.ruoyi.chat.service.chat.IChatService;
import org.ruoyi.common.chat.entity.chat.ChatCompletion;
import org.ruoyi.common.chat.entity.chat.Message;
import org.ruoyi.common.chat.openai.OpenAiStreamClient;
import org.ruoyi.common.chat.request.ChatRequest;
import org.ruoyi.domain.vo.ChatModelVo;
import org.ruoyi.service.IChatModelService;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.util.*;
/**
* 图片识别模型
*/
@Service
@Slf4j
public class ImageQianwenServiceImpl implements IChatService {
@Autowired
private IChatModelService chatModelService;
@SneakyThrows
@Override
public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) {
ChatModelVo chatModelVo = chatModelService.selectModelByCategory("image");
// 发送流式消息
MultiModalConversation conv = new MultiModalConversation();
MultiModalMessage systemMessage = MultiModalMessage.builder().role(Role.SYSTEM.getValue())
.content(Arrays.asList(
Collections.singletonMap("text",chatRequest.getSysPrompt()))).build();
// 获取用户消息内容
List<Message> messages = chatRequest.getMessages();
MultiModalMessage userMessage = null;
//漫长的格式转换
// 遍历消息列表，提取文本内容
if (messages != null && !messages.isEmpty()) {
Object content = messages.get(messages.size() - 1).getContent();
List<Map<String, Object>> contentList = new ArrayList<>();
StringBuilder textContent = new StringBuilder();
if (content instanceof List<?>) {
for (Object item : (List<?>) content) {
if (item instanceof Map<?, ?> mapItem) {
String type = (String) mapItem.get("type");
if ("text".equals(type)) {
String text = (String) mapItem.get("text");
if (text != null) {
textContent.append(text).append(" ");
}
} else if ("image_url".equals(type)) {
Map<String, String> imageUrl = (Map<String, String>) mapItem.get("image_url");
contentList.add(Collections.singletonMap("image", imageUrl.get("url")));
}
}
}
}
// 将拼接后的文本内容添加到 contentList
if (textContent.length() > 0) {
contentList.add(Collections.singletonMap("text", textContent.toString().trim()));
}
userMessage = MultiModalMessage.builder()
.role(Role.USER.getValue())
.content(contentList)
.build();
}
MultiModalConversationParam param = MultiModalConversationParam.builder()
.apiKey(chatModelVo.getApiKey())
.model(chatModelVo.getModelName())
.messages(Arrays.asList(systemMessage, userMessage))
.incrementalOutput(true)
.build();
try {
final QwenStreamingResponseBuilder responseBuilder = new QwenStreamingResponseBuilder(param.getModel(),param.getIncrementalOutput() );
conv.streamCall(param, new ResultCallback<>() {
@SneakyThrows
public void onEvent(MultiModalConversationResult result) {
String delta = responseBuilder.append(result);
if (Utils.isNotNullOrEmpty(delta)) {
emitter.send(delta);
log.info("收到消息片段: {}", delta);
}
}
public void onComplete() {
emitter.complete();
log.info("消息结束", responseBuilder.build());
}
public void onError(Exception e) {
log.info("请求失败", e.getMessage());
}
});
} catch (NoApiKeyException e) {
emitter.send("请先配置API密钥");
throw new RuntimeException(e);
} catch (UploadFileException e) {
throw new RuntimeException(e);
}
return emitter;
}
@Override
public String getCategory() {
return ChatModeType.IMAGE.getCode();
}
}

151
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageServiceImpl.java Normal file
View File

@@ -0,0 +1,151 @@
package org.ruoyi.chat.service.chat.impl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.chat.config.ChatConfig;
import org.ruoyi.chat.enums.ChatModeType;
import org.ruoyi.chat.listener.SSEEventSourceListener;
import org.ruoyi.chat.service.chat.IChatService;
import org.ruoyi.common.chat.entity.chat.ChatCompletion;
import org.ruoyi.common.chat.entity.chat.Message;
import org.ruoyi.common.chat.openai.OpenAiStreamClient;
import org.ruoyi.common.chat.request.ChatRequest;
import org.ruoyi.domain.vo.ChatModelVo;
import org.ruoyi.service.IChatModelService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.util.*;
/**
* 图片识别模型
*/
@Service
@Slf4j
public class ImageServiceImpl implements IChatService {
@Autowired
private IChatModelService chatModelService;
@SneakyThrows
// @Override
// public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) {
// ChatModelVo chatModelVo = chatModelService.selectModelByCategory("image");
//
// // 发送流式消息
//
// MultiModalConversation conv = new MultiModalConversation();
// MultiModalMessage systemMessage = MultiModalMessage.builder().role(Role.SYSTEM.getValue())
// .content(Arrays.asList(
// Collections.singletonMap("text",chatRequest.getSysPrompt()))).build();
// // 获取用户消息内容
// List<Message> messages = chatRequest.getMessages();
// MultiModalMessage userMessage = null;
// //漫长的格式转换
// // 遍历消息列表，提取文本内容
// if (messages != null && !messages.isEmpty()) {
// Object content = messages.get(messages.size() - 1).getContent();
// List<Map<String, Object>> contentList = new ArrayList<>();
// StringBuilder textContent = new StringBuilder();
// if (content instanceof List<?>) {
// for (Object item : (List<?>) content) {
// if (item instanceof Map<?, ?> mapItem) {
// String type = (String) mapItem.get("type");
// if ("text".equals(type)) {
// String text = (String) mapItem.get("text");
// if (text != null) {
// textContent.append(text).append(" ");
// }
// } else if ("image_url".equals(type)) {
// Map<String, String> imageUrl = (Map<String, String>) mapItem.get("image_url");
// contentList.add(Collections.singletonMap("image", imageUrl.get("url")));
// }
// }
// }
// }
// // 将拼接后的文本内容添加到 contentList
// if (textContent.length() > 0) {
// contentList.add(Collections.singletonMap("text", textContent.toString().trim()));
// }
// userMessage = MultiModalMessage.builder()
// .role(Role.USER.getValue())
// .content(contentList)
// .build();
// }
// MultiModalConversationParam param = MultiModalConversationParam.builder()
// .apiKey(chatModelVo.getApiKey())
// .model(chatModelVo.getModelName())
// .messages(Arrays.asList(systemMessage, userMessage))
// .incrementalOutput(true)
// .build();
//
//
// try {
// final QwenStreamingResponseBuilder responseBuilder = new QwenStreamingResponseBuilder(param.getModel(),param.getIncrementalOutput() );
// conv.streamCall(param, new ResultCallback<>() {
// @SneakyThrows
// public void onEvent(MultiModalConversationResult result) {
//
// String delta = responseBuilder.append(result);
// if (Utils.isNotNullOrEmpty(delta)) {
//
// emitter.send(delta);
// log.info("收到消息片段: {}", delta);
// }
// }
// public void onComplete() {
// emitter.complete();
// log.info("消息结束", responseBuilder.build());
// }
// public void onError(Exception e) {
// log.info("请求失败", e.getMessage());
// }
// });
// } catch (NoApiKeyException e) {
// emitter.send("请先配置API密钥");
// throw new RuntimeException(e);
// } catch (UploadFileException e) {
// throw new RuntimeException(e);
// }
//
//
// return emitter;
// }
@Override
public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) {
// 从数据库获取 image 类型的模型配置
ChatModelVo chatModelVo = chatModelService.selectModelByCategory(ChatModeType.IMAGE.getCode());
if (chatModelVo == null) {
log.error("未找到 image 类型的模型配置");
emitter.completeWithError(new IllegalStateException("未找到 image 类型的模型配置"));
return emitter;
}
// 创建 OpenAI 流客户端
OpenAiStreamClient openAiStreamClient = ChatConfig.createOpenAiStreamClient(chatModelVo.getApiHost(), chatModelVo.getApiKey());
List<Message> messages = chatRequest.getMessages();
// 创建 SSE 事件源监听器
SSEEventSourceListener listener = new SSEEventSourceListener(emitter, chatRequest.getUserId(), chatRequest.getSessionId());
// 构建聊天完成请求
ChatCompletion completion = ChatCompletion
.builder()
.messages(messages)
.model(chatModelVo.getModelName()) // 使用数据库中配置的模型名称
.stream(true)
.build();
// 发起流式聊天完成请求
openAiStreamClient.streamChatCompletion(completion, listener);
return emitter;
}
@Override
public String getCategory() {
return ChatModeType.IMAGE.getCode();
}
}

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/QianWenAiChatServiceImpl.java
View File

@@ -35,6 +35,8 @@ public class QianWenAiChatServiceImpl implements IChatService {
.modelName(chatModelVo.getModelName())
.build();
// 发送流式消息
try {
model.chat(chatRequest.getPrompt(), new StreamingChatResponseHandler() {

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java
View File

@@ -131,7 +131,7 @@ public class SseServiceImpl implements ISseService {
if (chatModelVo == null) {
log.error("未找到image类型的模型配置");
throw new IllegalStateException("未找到image类型的模型配置");
}// chatRequest.setModel(chatModelVo.getModelName());
}
}else{
chatModelVo = chatModelService.selectModelByName(chatRequest.getModel());
}

7
script/sql/update/20250605.sql Normal file
View File

@@ -0,0 +1,7 @@
/*
上传图片后会自动查找分类为image的模型,要使用图片识别功能,需要执行这条sql并配置key信息
*/
INSERT INTO `chat_model` (`id`, `tenant_id`, `category`, `model_name`, `model_describe`, `model_price`, `model_type`, `model_show`, `system_prompt`, `api_host`, `api_key`, `api_url`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (1930184891812147202, '000000', 'image', 'qwen/qwen2.5-vl-72b-instruct', 'qwen/qwen2.5-vl-72b-instruct', 0.003, '2', '0', NULL, 'https://api.ppinfra.com/v3/openai/chat/completions', 'xx', NULL, 103, 1, '2025-06-04 16:48:34', 1, '2025-06-04 16:48:34', '视觉模型');