From d635e30b4aa0ec38844f1094038193308dd8fdd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=92=E4=BA=A6?= Date: Mon, 2 Jun 2025 08:11:22 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=BD=BF=E7=94=A8=E5=90=8E=E5=8F=B0image?= =?UTF-8?q?=E5=88=86=E7=B1=BB=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/ruoyi/service/IChatModelService.java | 6 +- .../service/impl/ChatModelServiceImpl.java | 7 ++ .../org/ruoyi/chat/enums/ChatModeType.java | 4 +- .../chat/impl/ImageOpenAiServiceImpl.java | 75 +++++++++++++++++++ .../service/chat/impl/SseServiceImpl.java | 11 ++- 5 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageOpenAiServiceImpl.java diff --git a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/service/IChatModelService.java b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/service/IChatModelService.java index 86eef8dd..62dbf970 100644 --- a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/service/IChatModelService.java +++ b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/service/IChatModelService.java @@ -53,9 +53,13 @@ public interface IChatModelService { * 通过模型名称获取模型信息 */ ChatModelVo selectModelByName(String modelName); - + /** + * 通过模型分类获取模型信息 + */ + ChatModelVo selectModelByCategory(String image); /** * 获取ppt模型信息 */ ChatModel getPPT(); + } diff --git a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/service/impl/ChatModelServiceImpl.java b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/service/impl/ChatModelServiceImpl.java index 4c82f3df..d0b5f5cd 100644 --- a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/service/impl/ChatModelServiceImpl.java +++ b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/service/impl/ChatModelServiceImpl.java @@ -129,6 +129,13 @@ public class ChatModelServiceImpl implements IChatModelService { public ChatModelVo selectModelByName(String modelName) { return baseMapper.selectVoOne(Wrappers.lambdaQuery().eq(ChatModel::getModelName, modelName)); } + /** + * 通过模型分类获取模型信息 + */ + @Override + public ChatModelVo selectModelByCategory(String category) { + return baseMapper.selectVoOne(Wrappers.lambdaQuery().eq(ChatModel::getCategory, category)); + } @Override public ChatModel getPPT() { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/enums/ChatModeType.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/enums/ChatModeType.java index 8388ad2f..13a39d90 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/enums/ChatModeType.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/enums/ChatModeType.java @@ -15,7 +15,9 @@ public enum ChatModeType { QIANWEN("qianwen", "通义千问"), - VECTOR("vector", "知识库向量模型"); + VECTOR("vector", "知识库向量模型"), + + IMAGE("image", "图片识别模型"); private final String code; private final String description; diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageOpenAiServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageOpenAiServiceImpl.java new file mode 100644 index 00000000..11b4c1ec --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageOpenAiServiceImpl.java @@ -0,0 +1,75 @@ +package org.ruoyi.chat.service.chat.impl; + +import io.modelcontextprotocol.client.McpSyncClient; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.chat.config.ChatConfig; +import org.ruoyi.chat.enums.ChatModeType; +import org.ruoyi.chat.listener.SSEEventSourceListener; +import org.ruoyi.chat.service.chat.IChatService; +import org.ruoyi.common.chat.entity.chat.ChatCompletion; +import org.ruoyi.common.chat.entity.chat.Message; +import org.ruoyi.common.chat.openai.OpenAiStreamClient; +import org.ruoyi.common.chat.request.ChatRequest; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.service.IChatModelService; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import java.util.List; + +/** + * 图片识别模型 + */ +@Service +@Slf4j +public class ImageOpenAiServiceImpl implements IChatService { + + @Autowired + private IChatModelService chatModelService; + + private final ChatClient chatClient; + + public ImageOpenAiServiceImpl(ChatClient.Builder chatClientBuilder) { + this.chatClient = chatClientBuilder.build(); + } + + @Override + public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) { + // 从数据库获取 image 类型的模型配置 + ChatModelVo chatModelVo = chatModelService.selectModelByCategory(ChatModeType.IMAGE.getCode()); + if (chatModelVo == null) { + log.error("未找到 image 类型的模型配置"); + emitter.completeWithError(new IllegalStateException("未找到 image 类型的模型配置")); + return emitter; + } + + // 创建 OpenAI 流客户端 + OpenAiStreamClient openAiStreamClient = ChatConfig.createOpenAiStreamClient(chatModelVo.getApiHost(), chatModelVo.getApiKey()); + List messages = chatRequest.getMessages(); + + // 创建 SSE 事件源监听器 + SSEEventSourceListener listener = new SSEEventSourceListener(emitter, chatRequest.getUserId(), chatRequest.getSessionId()); + + // 构建聊天完成请求 + ChatCompletion completion = ChatCompletion + .builder() + .messages(messages) + .model(chatModelVo.getModelName()) // 使用数据库中配置的模型名称 + .stream(true) + .build(); + + // 发起流式聊天完成请求 + openAiStreamClient.streamChatCompletion(completion, listener); + + return emitter; + } + + + @Override + public String getCategory() { + return ChatModeType.IMAGE.getCode(); + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java index 472fd7b8..f7dee9a6 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java @@ -125,7 +125,16 @@ public class SseServiceImpl implements ISseService { */ private void buildChatMessageList(ChatRequest chatRequest){ String sysPrompt; - chatModelVo = chatModelService.selectModelByName(chatRequest.getModel()); + // 矫正模型名称 如果是gpt-image 则查询image类型模型 获取模型名称 + if(chatRequest.getModel().equals("gpt-image")) { + chatModelVo = chatModelService.selectModelByCategory("image"); + if (chatModelVo == null) { + log.error("未找到image类型的模型配置"); + throw new IllegalStateException("未找到image类型的模型配置"); + }// chatRequest.setModel(chatModelVo.getModelName()); + }else{ + chatModelVo = chatModelService.selectModelByName(chatRequest.getModel()); + } // 获取对话消息列表 List messages = chatRequest.getMessages(); // 查询向量库相关信息加入到上下文 From 0a73cb4e1725592470d446a9d2a09e95e27868ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=92=E4=BA=A6?= Date: Thu, 5 Jun 2025 12:05:28 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=BD=BF=E7=94=A8=E5=90=8E=E5=8F=B0image?= =?UTF-8?q?=E5=88=86=E7=B1=BB=E4=B8=8B=E9=80=9A=E4=B9=89=E5=8D=83=E9=97=AE?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chat/impl/ImageOpenAiServiceImpl.java | 75 ---------- .../chat/impl/ImageQianwenServiceImpl.java | 140 ++++++++++++++++++ 2 files changed, 140 insertions(+), 75 deletions(-) delete mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageOpenAiServiceImpl.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageOpenAiServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageOpenAiServiceImpl.java deleted file mode 100644 index 11b4c1ec..00000000 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageOpenAiServiceImpl.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.ruoyi.chat.service.chat.impl; - -import io.modelcontextprotocol.client.McpSyncClient; -import lombok.extern.slf4j.Slf4j; -import org.ruoyi.chat.config.ChatConfig; -import org.ruoyi.chat.enums.ChatModeType; -import org.ruoyi.chat.listener.SSEEventSourceListener; -import org.ruoyi.chat.service.chat.IChatService; -import org.ruoyi.common.chat.entity.chat.ChatCompletion; -import org.ruoyi.common.chat.entity.chat.Message; -import org.ruoyi.common.chat.openai.OpenAiStreamClient; -import org.ruoyi.common.chat.request.ChatRequest; -import org.ruoyi.domain.vo.ChatModelVo; -import org.ruoyi.service.IChatModelService; -import org.springframework.ai.chat.client.ChatClient; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; - -import java.util.List; - -/** - * 图片识别模型 - */ -@Service -@Slf4j -public class ImageOpenAiServiceImpl implements IChatService { - - @Autowired - private IChatModelService chatModelService; - - private final ChatClient chatClient; - - public ImageOpenAiServiceImpl(ChatClient.Builder chatClientBuilder) { - this.chatClient = chatClientBuilder.build(); - } - - @Override - public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) { - // 从数据库获取 image 类型的模型配置 - ChatModelVo chatModelVo = chatModelService.selectModelByCategory(ChatModeType.IMAGE.getCode()); - if (chatModelVo == null) { - log.error("未找到 image 类型的模型配置"); - emitter.completeWithError(new IllegalStateException("未找到 image 类型的模型配置")); - return emitter; - } - - // 创建 OpenAI 流客户端 - OpenAiStreamClient openAiStreamClient = ChatConfig.createOpenAiStreamClient(chatModelVo.getApiHost(), chatModelVo.getApiKey()); - List messages = chatRequest.getMessages(); - - // 创建 SSE 事件源监听器 - SSEEventSourceListener listener = new SSEEventSourceListener(emitter, chatRequest.getUserId(), chatRequest.getSessionId()); - - // 构建聊天完成请求 - ChatCompletion completion = ChatCompletion - .builder() - .messages(messages) - .model(chatModelVo.getModelName()) // 使用数据库中配置的模型名称 - .stream(true) - .build(); - - // 发起流式聊天完成请求 - openAiStreamClient.streamChatCompletion(completion, listener); - - return emitter; - } - - - @Override - public String getCategory() { - return ChatModeType.IMAGE.getCode(); - } -} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java new file mode 100644 index 00000000..aefd62f0 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java @@ -0,0 +1,140 @@ +package org.ruoyi.chat.service.chat.impl; + +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversation; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationParam; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationResult; +import com.alibaba.dashscope.common.MultiModalMessage; +import com.alibaba.dashscope.common.ResultCallback; +import com.alibaba.dashscope.common.Role; +import com.alibaba.dashscope.exception.NoApiKeyException; +import com.alibaba.dashscope.exception.UploadFileException; +import dev.langchain4j.community.model.dashscope.QwenStreamingChatModel; +import dev.langchain4j.community.model.dashscope.QwenStreamingResponseBuilder; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.internal.Utils; +import dev.langchain4j.model.chat.StreamingChatModel; +import dev.langchain4j.model.chat.response.ChatResponse; +import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; +import io.modelcontextprotocol.client.McpSyncClient; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.chat.config.ChatConfig; +import org.ruoyi.chat.enums.ChatModeType; +import org.ruoyi.chat.listener.SSEEventSourceListener; +import org.ruoyi.chat.service.chat.IChatService; +import org.ruoyi.common.chat.entity.chat.ChatCompletion; +import org.ruoyi.common.chat.entity.chat.Message; +import org.ruoyi.common.chat.openai.OpenAiStreamClient; +import org.ruoyi.common.chat.request.ChatRequest; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.service.IChatModelService; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import java.util.*; + +/** + * 图片识别模型 + */ +@Service +@Slf4j +public class ImageQianwenServiceImpl implements IChatService { + + @Autowired + private IChatModelService chatModelService; + + @SneakyThrows + @Override + public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) { + ChatModelVo chatModelVo = chatModelService.selectModelByCategory("image"); + + // 发送流式消息 + + MultiModalConversation conv = new MultiModalConversation(); + MultiModalMessage systemMessage = MultiModalMessage.builder().role(Role.SYSTEM.getValue()) + .content(Arrays.asList( + Collections.singletonMap("text",chatRequest.getSysPrompt()))).build(); + // 获取用户消息内容 + List messages = chatRequest.getMessages(); + MultiModalMessage userMessage = null; + //漫长的格式转换 + // 遍历消息列表，提取文本内容 + if (messages != null && !messages.isEmpty()) { + Object content = messages.get(messages.size() - 1).getContent(); + List> contentList = new ArrayList<>(); + StringBuilder textContent = new StringBuilder(); + if (content instanceof List) { + for (Object item : (List) content) { + if (item instanceof Map mapItem) { + String type = (String) mapItem.get("type"); + if ("text".equals(type)) { + String text = (String) mapItem.get("text"); + if (text != null) { + textContent.append(text).append(" "); + } + } else if ("image_url".equals(type)) { + Map imageUrl = (Map) mapItem.get("image_url"); + contentList.add(Collections.singletonMap("image", imageUrl.get("url"))); + } + } + } + } + // 将拼接后的文本内容添加到 contentList + if (textContent.length() > 0) { + contentList.add(Collections.singletonMap("text", textContent.toString().trim())); + } + userMessage = MultiModalMessage.builder() + .role(Role.USER.getValue()) + .content(contentList) + .build(); + } + MultiModalConversationParam param = MultiModalConversationParam.builder() + .apiKey(chatModelVo.getApiKey()) + .model(chatModelVo.getModelName()) + .messages(Arrays.asList(systemMessage, userMessage)) + .incrementalOutput(true) + .build(); + + + try { + final QwenStreamingResponseBuilder responseBuilder = new QwenStreamingResponseBuilder(param.getModel(),param.getIncrementalOutput() ); + conv.streamCall(param, new ResultCallback<>() { + @SneakyThrows + public void onEvent(MultiModalConversationResult result) { + + String delta = responseBuilder.append(result); + if (Utils.isNotNullOrEmpty(delta)) { + + emitter.send(delta); + log.info("收到消息片段: {}", delta); + } + } + public void onComplete() { + emitter.complete(); + log.info("消息结束", responseBuilder.build()); + } + public void onError(Exception e) { + log.info("请求失败", e.getMessage()); + } + }); + } catch (NoApiKeyException e) { + emitter.send("请先配置API密钥"); + throw new RuntimeException(e); + } catch (UploadFileException e) { + throw new RuntimeException(e); + } + + + return emitter; + } + @Override + public String getCategory() { + return ChatModeType.IMAGE.getCode(); + } +} From baa664ac4fd1bf72888a59f1099edcf20f22b74f Mon Sep 17 00:00:00 2001 From: ageerle Date: Thu, 5 Jun 2025 16:00:06 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20=E5=9B=BE=E7=89=87=E8=AF=86?= =?UTF-8?q?=E5=88=AB=E5=8A=9F=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chat/impl/ImageQianwenServiceImpl.java | 140 ---------------- .../service/chat/impl/ImageServiceImpl.java | 151 ++++++++++++++++++ .../chat/impl/QianWenAiChatServiceImpl.java | 2 + .../service/chat/impl/SseServiceImpl.java | 2 +- script/sql/update/20250605.sql | 7 + 5 files changed, 161 insertions(+), 141 deletions(-) delete mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageServiceImpl.java create mode 100644 script/sql/update/20250605.sql diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java deleted file mode 100644 index aefd62f0..00000000 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java +++ /dev/null @@ -1,140 +0,0 @@ -package org.ruoyi.chat.service.chat.impl; - -import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversation; -import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationParam; -import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationResult; -import com.alibaba.dashscope.common.MultiModalMessage; -import com.alibaba.dashscope.common.ResultCallback; -import com.alibaba.dashscope.common.Role; -import com.alibaba.dashscope.exception.NoApiKeyException; -import com.alibaba.dashscope.exception.UploadFileException; -import dev.langchain4j.community.model.dashscope.QwenStreamingChatModel; -import dev.langchain4j.community.model.dashscope.QwenStreamingResponseBuilder; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.SystemMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.internal.Utils; -import dev.langchain4j.model.chat.StreamingChatModel; -import dev.langchain4j.model.chat.response.ChatResponse; -import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; -import io.modelcontextprotocol.client.McpSyncClient; -import io.reactivex.Flowable; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.ruoyi.chat.config.ChatConfig; -import org.ruoyi.chat.enums.ChatModeType; -import org.ruoyi.chat.listener.SSEEventSourceListener; -import org.ruoyi.chat.service.chat.IChatService; -import org.ruoyi.common.chat.entity.chat.ChatCompletion; -import org.ruoyi.common.chat.entity.chat.Message; -import org.ruoyi.common.chat.openai.OpenAiStreamClient; -import org.ruoyi.common.chat.request.ChatRequest; -import org.ruoyi.domain.vo.ChatModelVo; -import org.ruoyi.service.IChatModelService; -import org.springframework.ai.chat.client.ChatClient; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; - -import java.util.*; - -/** - * 图片识别模型 - */ -@Service -@Slf4j -public class ImageQianwenServiceImpl implements IChatService { - - @Autowired - private IChatModelService chatModelService; - - @SneakyThrows - @Override - public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) { - ChatModelVo chatModelVo = chatModelService.selectModelByCategory("image"); - - // 发送流式消息 - - MultiModalConversation conv = new MultiModalConversation(); - MultiModalMessage systemMessage = MultiModalMessage.builder().role(Role.SYSTEM.getValue()) - .content(Arrays.asList( - Collections.singletonMap("text",chatRequest.getSysPrompt()))).build(); - // 获取用户消息内容 - List messages = chatRequest.getMessages(); - MultiModalMessage userMessage = null; - //漫长的格式转换 - // 遍历消息列表，提取文本内容 - if (messages != null && !messages.isEmpty()) { - Object content = messages.get(messages.size() - 1).getContent(); - List> contentList = new ArrayList<>(); - StringBuilder textContent = new StringBuilder(); - if (content instanceof List) { - for (Object item : (List) content) { - if (item instanceof Map mapItem) { - String type = (String) mapItem.get("type"); - if ("text".equals(type)) { - String text = (String) mapItem.get("text"); - if (text != null) { - textContent.append(text).append(" "); - } - } else if ("image_url".equals(type)) { - Map imageUrl = (Map) mapItem.get("image_url"); - contentList.add(Collections.singletonMap("image", imageUrl.get("url"))); - } - } - } - } - // 将拼接后的文本内容添加到 contentList - if (textContent.length() > 0) { - contentList.add(Collections.singletonMap("text", textContent.toString().trim())); - } - userMessage = MultiModalMessage.builder() - .role(Role.USER.getValue()) - .content(contentList) - .build(); - } - MultiModalConversationParam param = MultiModalConversationParam.builder() - .apiKey(chatModelVo.getApiKey()) - .model(chatModelVo.getModelName()) - .messages(Arrays.asList(systemMessage, userMessage)) - .incrementalOutput(true) - .build(); - - - try { - final QwenStreamingResponseBuilder responseBuilder = new QwenStreamingResponseBuilder(param.getModel(),param.getIncrementalOutput() ); - conv.streamCall(param, new ResultCallback<>() { - @SneakyThrows - public void onEvent(MultiModalConversationResult result) { - - String delta = responseBuilder.append(result); - if (Utils.isNotNullOrEmpty(delta)) { - - emitter.send(delta); - log.info("收到消息片段: {}", delta); - } - } - public void onComplete() { - emitter.complete(); - log.info("消息结束", responseBuilder.build()); - } - public void onError(Exception e) { - log.info("请求失败", e.getMessage()); - } - }); - } catch (NoApiKeyException e) { - emitter.send("请先配置API密钥"); - throw new RuntimeException(e); - } catch (UploadFileException e) { - throw new RuntimeException(e); - } - - - return emitter; - } - @Override - public String getCategory() { - return ChatModeType.IMAGE.getCode(); - } -} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageServiceImpl.java new file mode 100644 index 00000000..a7eee8e6 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageServiceImpl.java @@ -0,0 +1,151 @@ +package org.ruoyi.chat.service.chat.impl; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.chat.config.ChatConfig; +import org.ruoyi.chat.enums.ChatModeType; +import org.ruoyi.chat.listener.SSEEventSourceListener; +import org.ruoyi.chat.service.chat.IChatService; +import org.ruoyi.common.chat.entity.chat.ChatCompletion; +import org.ruoyi.common.chat.entity.chat.Message; +import org.ruoyi.common.chat.openai.OpenAiStreamClient; +import org.ruoyi.common.chat.request.ChatRequest; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.service.IChatModelService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import java.util.*; + +/** + * 图片识别模型 + */ +@Service +@Slf4j +public class ImageServiceImpl implements IChatService { + + @Autowired + private IChatModelService chatModelService; + + @SneakyThrows +// @Override +// public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) { +// ChatModelVo chatModelVo = chatModelService.selectModelByCategory("image"); +// +// // 发送流式消息 +// +// MultiModalConversation conv = new MultiModalConversation(); +// MultiModalMessage systemMessage = MultiModalMessage.builder().role(Role.SYSTEM.getValue()) +// .content(Arrays.asList( +// Collections.singletonMap("text",chatRequest.getSysPrompt()))).build(); +// // 获取用户消息内容 +// List messages = chatRequest.getMessages(); +// MultiModalMessage userMessage = null; +// //漫长的格式转换 +// // 遍历消息列表，提取文本内容 +// if (messages != null && !messages.isEmpty()) { +// Object content = messages.get(messages.size() - 1).getContent(); +// List> contentList = new ArrayList<>(); +// StringBuilder textContent = new StringBuilder(); +// if (content instanceof List) { +// for (Object item : (List) content) { +// if (item instanceof Map mapItem) { +// String type = (String) mapItem.get("type"); +// if ("text".equals(type)) { +// String text = (String) mapItem.get("text"); +// if (text != null) { +// textContent.append(text).append(" "); +// } +// } else if ("image_url".equals(type)) { +// Map imageUrl = (Map) mapItem.get("image_url"); +// contentList.add(Collections.singletonMap("image", imageUrl.get("url"))); +// } +// } +// } +// } +// // 将拼接后的文本内容添加到 contentList +// if (textContent.length() > 0) { +// contentList.add(Collections.singletonMap("text", textContent.toString().trim())); +// } +// userMessage = MultiModalMessage.builder() +// .role(Role.USER.getValue()) +// .content(contentList) +// .build(); +// } +// MultiModalConversationParam param = MultiModalConversationParam.builder() +// .apiKey(chatModelVo.getApiKey()) +// .model(chatModelVo.getModelName()) +// .messages(Arrays.asList(systemMessage, userMessage)) +// .incrementalOutput(true) +// .build(); +// +// +// try { +// final QwenStreamingResponseBuilder responseBuilder = new QwenStreamingResponseBuilder(param.getModel(),param.getIncrementalOutput() ); +// conv.streamCall(param, new ResultCallback<>() { +// @SneakyThrows +// public void onEvent(MultiModalConversationResult result) { +// +// String delta = responseBuilder.append(result); +// if (Utils.isNotNullOrEmpty(delta)) { +// +// emitter.send(delta); +// log.info("收到消息片段: {}", delta); +// } +// } +// public void onComplete() { +// emitter.complete(); +// log.info("消息结束", responseBuilder.build()); +// } +// public void onError(Exception e) { +// log.info("请求失败", e.getMessage()); +// } +// }); +// } catch (NoApiKeyException e) { +// emitter.send("请先配置API密钥"); +// throw new RuntimeException(e); +// } catch (UploadFileException e) { +// throw new RuntimeException(e); +// } +// +// +// return emitter; +// } + + @Override + public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) { + // 从数据库获取 image 类型的模型配置 + ChatModelVo chatModelVo = chatModelService.selectModelByCategory(ChatModeType.IMAGE.getCode()); + if (chatModelVo == null) { + log.error("未找到 image 类型的模型配置"); + emitter.completeWithError(new IllegalStateException("未找到 image 类型的模型配置")); + return emitter; + } + + // 创建 OpenAI 流客户端 + OpenAiStreamClient openAiStreamClient = ChatConfig.createOpenAiStreamClient(chatModelVo.getApiHost(), chatModelVo.getApiKey()); + List messages = chatRequest.getMessages(); + + // 创建 SSE 事件源监听器 + SSEEventSourceListener listener = new SSEEventSourceListener(emitter, chatRequest.getUserId(), chatRequest.getSessionId()); + + // 构建聊天完成请求 + ChatCompletion completion = ChatCompletion + .builder() + .messages(messages) + .model(chatModelVo.getModelName()) // 使用数据库中配置的模型名称 + .stream(true) + .build(); + + // 发起流式聊天完成请求 + openAiStreamClient.streamChatCompletion(completion, listener); + + return emitter; + } + + @Override + public String getCategory() { + return ChatModeType.IMAGE.getCode(); + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/QianWenAiChatServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/QianWenAiChatServiceImpl.java index 1d6e061f..850ebf6c 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/QianWenAiChatServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/QianWenAiChatServiceImpl.java @@ -35,6 +35,8 @@ public class QianWenAiChatServiceImpl implements IChatService { .modelName(chatModelVo.getModelName()) .build(); + + // 发送流式消息 try { model.chat(chatRequest.getPrompt(), new StreamingChatResponseHandler() { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java index f7dee9a6..d8c61a9c 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java @@ -131,7 +131,7 @@ public class SseServiceImpl implements ISseService { if (chatModelVo == null) { log.error("未找到image类型的模型配置"); throw new IllegalStateException("未找到image类型的模型配置"); - }// chatRequest.setModel(chatModelVo.getModelName()); + } }else{ chatModelVo = chatModelService.selectModelByName(chatRequest.getModel()); } diff --git a/script/sql/update/20250605.sql b/script/sql/update/20250605.sql new file mode 100644 index 00000000..3d2120d8 --- /dev/null +++ b/script/sql/update/20250605.sql @@ -0,0 +1,7 @@ + +/* + 上传图片后会自动查找分类为image的模型,要使用图片识别功能,需要执行这条sql并配置key信息 +*/ + + +INSERT INTO `chat_model` (`id`, `tenant_id`, `category`, `model_name`, `model_describe`, `model_price`, `model_type`, `model_show`, `system_prompt`, `api_host`, `api_key`, `api_url`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (1930184891812147202, '000000', 'image', 'qwen/qwen2.5-vl-72b-instruct', 'qwen/qwen2.5-vl-72b-instruct', 0.003, '2', '0', NULL, 'https://api.ppinfra.com/v3/openai/chat/completions', 'xx', NULL, 103, 1, '2025-06-04 16:48:34', 1, '2025-06-04 16:48:34', '视觉模型'); From 97ae5a46cd54b2d5b10fca6a69c04874002162b0 Mon Sep 17 00:00:00 2001 From: ageerle Date: Thu, 5 Jun 2025 16:16:32 +0800 Subject: [PATCH 4/4] =?UTF-8?q?feat:=20=E8=B0=83=E6=95=B4sql=E8=84=9A?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/sql/ruoyi-ai.sql | 4 ++-- script/sql/update/20250605.sql | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/script/sql/ruoyi-ai.sql b/script/sql/ruoyi-ai.sql index ed5eb310..f4356fd2 100644 --- a/script/sql/ruoyi-ai.sql +++ b/script/sql/ruoyi-ai.sql @@ -132,8 +132,8 @@ INSERT INTO `chat_model` VALUES (1828324413241466880, '000000', 'vector', 'quent INSERT INTO `chat_model` VALUES (1828324413241466881, '000000', 'vector', 'baai/bge-m3', 'baai/bge-m3', 0.01, '2', '1', NULL, 'https://api.ppinfra.com/v3/openai', 'sk-xx', NULL, 103, 1, '2024-08-27 14:51:23', 1, '2025-05-24 17:33:11', 'BGE-M3 是一款具备多维度能力的文本嵌入模型，可同时实现密集检索、多向量检索和稀疏检索三大核心功能。该模型设计上兼容超过100种语言，并支持从短句到长达8192词元的长文本等多种输入形式。在跨语言检索任务中，BGE-M3展现出显著优势，其性能在MIRACL、MKQA等国际基准测试中位居前列。此外，针对长文档检索场景，该模型在MLDR、NarritiveQA等数据集上的表现同样达到行业领先水平。'); INSERT INTO `chat_model` VALUES (1859570229117022211, '000000', 'chat', 'deepseek/deepseek-v3-0324', 'deepseek/deepseek-v3-0324', 0.1, '1', '0', '', 'https://api.ppinfra.com/v3/openai/chat/completions', 'sk-xx', NULL, 103, 1, '2024-11-21 20:11:06', 1, '2025-05-24 17:56:22', 'DeepSeek V3 0324 是深度求索（DeepSeek）团队旗舰级对话模型系列的最新版本，采用混合专家（Mixture-of-Experts, MoE）架构，参数量达685B参数。'); INSERT INTO `chat_model` VALUES (1859570229117022212, '000000', 'chat', 'deepseek/deepseek-r1', 'deepseek/deepseek-r1', 0.1, '1', '0', '', 'https://api.ppinfra.com/v3/openai/chat/completions', 'sk-xx', NULL, 103, 1, '2024-11-21 20:11:06', 1, '2025-05-24 17:56:14', 'DeepSeek R1是DeepSeek团队发布的最新开源模型，具备非常强悍的推理性能，尤其在数学、编程和推理任务上达到了与OpenAI的o1模型相当的水平。'); -INSERT INTO `chat_model` VALUES (1926215622017482754, '000000', 'chat', 'gpt-4o-mini', 'gpt-4o-mini', 0.1, '1', '0', NULL, 'https://api.pandarobot.chat/v1/chat/completions/', 'sk-xx', NULL, 103, 1, '2025-05-24 17:56:06', 1, '2025-05-24 17:56:06', 'gpt 多模态模型'); -INSERT INTO `chat_model` VALUES (1926215622017482755, '000000', 'chat', 'gpt-4-all', 'gpt-4-all', 0.5, '2', '0', NULL, 'https://api.pandarobot.chat/v1/chat/completions/', 'sk-xx', NULL, 103, 1, '2025-05-24 17:56:06', 1, '2025-05-24 17:59:21', 'gpt 逆向多模态模型'); +INSERT INTO `chat_model` VALUES (1930184891812147202, '000000', 'image', 'qwen/qwen2.5-vl-72b-instruct', 'qwen/qwen2.5-vl-72b-instruct', 0.003, '2', '0', NULL, 'https://api.ppinfra.com/v3/openai/chat/completions', 'xx', NULL, 103, 1, '2025-06-04 16:48:34', 1, '2025-06-04 16:48:34', '视觉模型'); + -- ---------------------------- -- Table structure for chat_pay_order diff --git a/script/sql/update/20250605.sql b/script/sql/update/20250605.sql index 3d2120d8..8b137891 100644 --- a/script/sql/update/20250605.sql +++ b/script/sql/update/20250605.sql @@ -1,7 +1 @@ -/* - 上传图片后会自动查找分类为image的模型,要使用图片识别功能,需要执行这条sql并配置key信息 -*/ - - -INSERT INTO `chat_model` (`id`, `tenant_id`, `category`, `model_name`, `model_describe`, `model_price`, `model_type`, `model_show`, `system_prompt`, `api_host`, `api_key`, `api_url`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (1930184891812147202, '000000', 'image', 'qwen/qwen2.5-vl-72b-instruct', 'qwen/qwen2.5-vl-72b-instruct', 0.003, '2', '0', NULL, 'https://api.ppinfra.com/v3/openai/chat/completions', 'xx', NULL, 103, 1, '2025-06-04 16:48:34', 1, '2025-06-04 16:48:34', '视觉模型');