From b47da3f4383f48d53c9ab3d0cc30f1de69e7939c Mon Sep 17 00:00:00 2001 From: Robust_H <1511209518@qq.com> Date: Sat, 4 Oct 2025 04:50:12 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=88=9D=E5=A7=8B=E5=8C=96=E5=A4=9A?= =?UTF-8?q?=E4=BE=9B=E5=BA=94=E5=95=86=E5=A4=9A=E5=B5=8C=E5=85=A5=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E9=9B=86=E6=88=90=EF=BC=8C=E9=87=87=E7=94=A8=E7=AD=96?= =?UTF-8?q?=E7=95=A5=E6=A8=A1=E5=BC=8F=E5=92=8C=E5=B7=A5=E5=8E=82=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/org/ruoyi/domain/ChatModel.java | 6 + .../java/org/ruoyi/domain/bo/ChatModelBo.java | 5 + .../java/org/ruoyi/domain/vo/ChatModelVo.java | 6 + ruoyi-modules-api/ruoyi-knowledge-api/pom.xml | 4 + .../embedding/BaseEmbedModelService.java | 26 ++ .../embedding/EmbeddingModelFactory.java | 70 +++++ .../MultiModalEmbedModelService.java | 35 +++ .../impl/AliBaiLianBaseEmbedProvider.java | 14 + .../AliBaiLianMultiEmbeddingProvider.java | 281 ++++++++++++++++++ .../impl/OllamaEmbeddingProvider.java | 41 +++ .../impl/OpenAiEmbeddingProvider.java | 43 +++ .../impl/SiliconFlowEmbeddingProvider.java | 18 ++ .../impl/ZhiPuAiEmbeddingProvider.java | 43 +++ .../model/AliyunMultiModalEmbedRequest.java | 44 +++ .../model/AliyunMultiModalEmbedResponse.java | 44 +++ .../ruoyi/embedding/model/ModalityType.java | 8 + .../embedding/model/MultiModalInput.java | 71 +++++ 17 files changed, 759 insertions(+) create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/BaseEmbedModelService.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/EmbeddingModelFactory.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/MultiModalEmbedModelService.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianBaseEmbedProvider.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianMultiEmbeddingProvider.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OllamaEmbeddingProvider.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OpenAiEmbeddingProvider.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/SiliconFlowEmbeddingProvider.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/ZhiPuAiEmbeddingProvider.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedRequest.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedResponse.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/ModalityType.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/MultiModalInput.java diff --git a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/ChatModel.java b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/ChatModel.java index e8f0e308..34465613 100644 --- a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/ChatModel.java +++ b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/ChatModel.java @@ -1,6 +1,7 @@ package org.ruoyi.domain; +import com.alibaba.excel.annotation.ExcelProperty; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; @@ -81,6 +82,11 @@ public class ChatModel extends BaseEntity { */ private Integer priority; + /** + * 模型供应商 + */ + private String ProviderName; + /** * 备注 */ diff --git a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/bo/ChatModelBo.java b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/bo/ChatModelBo.java index b828515b..34ee975e 100644 --- a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/bo/ChatModelBo.java +++ b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/bo/ChatModelBo.java @@ -1,5 +1,6 @@ package org.ruoyi.domain.bo; +import com.alibaba.excel.annotation.ExcelProperty; import io.github.linpeilie.annotations.AutoMapper; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; @@ -85,6 +86,10 @@ public class ChatModelBo extends BaseEntity { @NotBlank(message = "密钥不能为空", groups = { AddGroup.class, EditGroup.class }) private String apiKey; + /** + * 模型供应商 + */ + private String ProviderName; /** * 备注 diff --git a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/vo/ChatModelVo.java b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/vo/ChatModelVo.java index 0638c13a..6a2de3cf 100644 --- a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/vo/ChatModelVo.java +++ b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/vo/ChatModelVo.java @@ -95,6 +95,12 @@ public class ChatModelVo implements Serializable { @ExcelProperty(value = "优先级") private Integer priority; + /** + * 模型供应商 + */ + @ExcelProperty(value = "模型供应商") + private String ProviderName; + /** * 备注 */ diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml b/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml index f8082a67..0394b6d1 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml +++ b/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml @@ -106,6 +106,10 @@ dashscope-sdk-java 2.19.0 + + org.ruoyi + ruoyi-chat-api + diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/BaseEmbedModelService.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/BaseEmbedModelService.java new file mode 100644 index 00000000..9b3d0021 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/BaseEmbedModelService.java @@ -0,0 +1,26 @@ +package org.ruoyi.embedding; + +import dev.langchain4j.model.embedding.EmbeddingModel; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.model.ModalityType; + +import java.util.Set; + +/** + * BaseEmbedModelService 接口，扩展了 EmbeddingModel 接口 + * 该接口定义了嵌入模型服务的基本配置和功能方法 + */ +public interface BaseEmbedModelService extends EmbeddingModel { + /** + * 根据配置信息配置嵌入模型 + * @param config 包含模型配置信息的 ChatModelVo 对象 + */ + void configure(ChatModelVo config); + + /** + * 获取当前嵌入模型支持的所有模态类型 + * @return 返回支持的模态类型集合 + */ + Set getSupportedModalities(); + +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/EmbeddingModelFactory.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/EmbeddingModelFactory.java new file mode 100644 index 00000000..0d2155cc --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/EmbeddingModelFactory.java @@ -0,0 +1,70 @@ +package org.ruoyi.embedding; + +import lombok.RequiredArgsConstructor; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.service.IChatModelService; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 嵌入模型工厂服务类 + * 负责创建和管理各种嵌入模型实例 + */ +@Service +@RequiredArgsConstructor +public class EmbeddingModelFactory { + private final ApplicationContext applicationContext; + + private final IChatModelService iChatModelService; + + private final Map modelCache = new ConcurrentHashMap<>(); + + public BaseEmbedModelService createModel(Long embeddingModelId) { + ChatModelVo chatModelVo = iChatModelService.queryById(embeddingModelId); + + return createModelInstance(chatModelVo.getProviderName(), chatModelVo); + } + + private BaseEmbedModelService createModelInstance(String factory, ChatModelVo config) { + try { + BaseEmbedModelService model = applicationContext.getBean(factory, BaseEmbedModelService.class); + // TODO 缓存设置 + model.configure(config); + + return model; + } catch (NoSuchBeanDefinitionException e) { + throw new IllegalArgumentException("获取不到嵌入模型: " + factory, e); + } + } + + // 检查模型是否支持多模态 + public boolean isMultimodalModel(Long tenantId) { + BaseEmbedModelService model = createModel(tenantId); + return model instanceof MultiModalEmbedModelService; + } + + // 获取多模态模型（如果支持） + public MultiModalEmbedModelService createMultimodalModel(Long tenantId) { + BaseEmbedModelService model = createModel(tenantId); + if (model instanceof MultiModalEmbedModelService) { + return (MultiModalEmbedModelService) model; + } + throw new IllegalArgumentException("该模型不支持多模态"); + } + + public void refreshModel(String tenantId, String factory) { + String cacheKey = tenantId + ":" + factory; + modelCache.remove(cacheKey); + } + + public List getSupportedFactories() { + return new ArrayList<>(applicationContext.getBeansOfType(BaseEmbedModelService.class) + .keySet()); + } +} \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/MultiModalEmbedModelService.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/MultiModalEmbedModelService.java new file mode 100644 index 00000000..062ff00f --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/MultiModalEmbedModelService.java @@ -0,0 +1,35 @@ +package org.ruoyi.embedding; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.model.output.Response; +import org.ruoyi.embedding.model.MultiModalInput; + + +/** + * 多模态嵌入模型服务接口，继承自基础嵌入模型服务 + * 该接口提供了处理图像、视频以及多模态数据并转换为嵌入向量的功能 + */ +public interface MultiModalEmbedModelService extends BaseEmbedModelService { + /** + * 将图像数据转换为嵌入向量 + * @param imageDataUrl 图像的地址，必须是公开可访问的URL + * @return 包含嵌入向量的响应对象，可能包含状态信息和嵌入结果 + */ + Response embedImage(String imageDataUrl); + + /** + * 将视频数据转换为嵌入向量 + * @param videoDataUrl 视频的地址，必须是公开可访问的URL + * @return 包含嵌入向量的响应对象，可能包含状态信息和嵌入结果 + */ + Response embedVideo(String videoDataUrl); + + + /** + * 处理多模态输入并返回嵌入向量的方法 + * + * @param input 包含多种模态信息（如图像、文本等）的输入对象 + * @return Response 包含嵌入向量的响应对象，Embedding通常表示输入数据的向量表示 + */ + Response embedMultiModal(MultiModalInput input); +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianBaseEmbedProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianBaseEmbedProvider.java new file mode 100644 index 00000000..1511a0fe --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianBaseEmbedProvider.java @@ -0,0 +1,14 @@ +package org.ruoyi.embedding.impl; + + +import org.springframework.stereotype.Component; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午3:00 + * @Description: 阿里百炼基础嵌入模型（兼容openai） + */ +@Component("alibailian") +public class AliBaiLianBaseEmbedProvider extends OpenAiEmbeddingProvider{ + +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianMultiEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianMultiEmbeddingProvider.java new file mode 100644 index 00000000..ad3e8374 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianMultiEmbeddingProvider.java @@ -0,0 +1,281 @@ +package org.ruoyi.embedding.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.output.Response; +import dev.langchain4j.model.output.TokenUsage; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.MultiModalEmbedModelService; +import org.ruoyi.embedding.model.AliyunMultiModalEmbedRequest; +import org.ruoyi.embedding.model.AliyunMultiModalEmbedResponse; +import org.ruoyi.embedding.model.ModalityType; +import org.ruoyi.embedding.model.MultiModalInput; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * 阿里云百炼多模态嵌入模型服务实现类 + * 实现了MultiModalEmbedModelService接口，提供文本、图像和视频的嵌入向量生成服务 + */ +@Component("bailianMultiModel") +@Slf4j +public class AliBaiLianMultiEmbeddingProvider implements MultiModalEmbedModelService { + private ChatModelVo chatModelVo; + + private final OkHttpClient okHttpClient; + + /** + * 构造函数，初始化HTTP客户端 + * 设置连接超时、读取超时和写入超时时间 + */ + public AliBaiLianMultiEmbeddingProvider() { + this.okHttpClient = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build(); + } + + /** + * 图像嵌入向量生成 + * @param imageDataUrl 图像数据的URL + * @return 包含图像嵌入向量的Response对象 + */ + @Override + public Response embedImage(String imageDataUrl) { + return embedSingleModality("image", imageDataUrl); + } + + /** + * 视频嵌入向量生成 + * @param videoDataUrl 视频数据的URL + * @return 包含视频嵌入向量的Response对象 + */ + @Override + public Response embedVideo(String videoDataUrl) { + return embedSingleModality("video", videoDataUrl); + } + + /** + * 多模态嵌入向量生成 + * 支持同时处理文本、图像和视频等多种模态的数据 + * @param input 包含多种模态输入的对象 + * @return 包含多模态嵌入向量的Response对象 + */ + @Override + public Response embedMultiModal(MultiModalInput input) { + try { + // 构建请求内容 + List> contents = buildContentMap(input); + if (contents.isEmpty()) { + throw new IllegalArgumentException("至少提供一种模态的内容"); + } + + // 构建请求 + AliyunMultiModalEmbedRequest request = buildRequest(contents, chatModelVo); + AliyunMultiModalEmbedResponse resp = executeRequest(request, chatModelVo); + + // 转换为 embeddings + Response> response = toEmbeddings(resp); + List embeddings = response.content(); + + if (embeddings.isEmpty()) { + log.warn("阿里云混合模态嵌入返回为空"); + return Response.from(Embedding.from(new float[0]), response.tokenUsage()); + } + + // 多模态通常取第一个向量作为代表，也可以根据业务场景返回多个 + return Response.from(embeddings.get(0), response.tokenUsage()); + + } catch (Exception e) { + log.error("阿里云混合模态嵌入失败", e); + throw new IllegalArgumentException("阿里云混合模态嵌入失败", e); + } + } + + /** + * 配置模型参数 + * @param config 模型配置信息 + */ + @Override + public void configure(ChatModelVo config) { + this.chatModelVo = config; + } + + /** + * 获取支持的模态类型 + * @return 支持的模态类型集合 + */ + @Override + public Set getSupportedModalities() { + return Set.of(ModalityType.TEXT, ModalityType.VIDEO, ModalityType.IMAGE); + } + + /** + * 批量文本嵌入向量生成 + * @param textSegments 文本段列表 + * @return 包含所有文本嵌入向量的Response对象 + */ + @Override + public Response> embedAll(List textSegments) { + if (textSegments.isEmpty()) return Response.from(Collections.emptyList()); + + try { + List> contents = new ArrayList<>(); + for (TextSegment segment : textSegments) { + contents.add(Map.of("text", segment.text())); + } + + AliyunMultiModalEmbedRequest request = buildRequest(contents, chatModelVo); + AliyunMultiModalEmbedResponse resp = executeRequest(request, chatModelVo); + + return toEmbeddings(resp); + } catch (Exception e) { + log.error("阿里云文本嵌入失败", e); + throw new IllegalArgumentException("阿里云文本嵌入失败", e); + } + } + + /** + * 单模态嵌入（图片/视频/单条文本）复用方法 + * @param key 模态类型（image/video/text） + * @param dataUrl 数据URL + * @return 包含嵌入向量的Response对象 + */ + + public Response embedSingleModality(String key, String dataUrl) { + try { + AliyunMultiModalEmbedRequest request = buildRequest(List.of(Map.of(key, dataUrl)), chatModelVo); + AliyunMultiModalEmbedResponse resp = executeRequest(request, chatModelVo); + + Response> response = toEmbeddings(resp); + List embeddings = response.content(); + + if (embeddings.isEmpty()) { + log.warn("阿里云 {} 嵌入返回为空", key); + return Response.from(Embedding.from(new float[0]), response.tokenUsage()); + } + + return Response.from(embeddings.get(0), response.tokenUsage()); + } catch (Exception e) { + log.error("阿里云 {} 嵌入失败", key, e); + throw new IllegalArgumentException("阿里云 " + key + " 嵌入失败", e); + } + } + + /** + * 构建请求对象 + * @param contents 请求内容列表 + * @param chatModelVo 模型配置信息 + * @return 构建好的请求对象 + */ + private AliyunMultiModalEmbedRequest buildRequest(List> contents, ChatModelVo chatModelVo) { + if (contents.isEmpty()) throw new IllegalArgumentException("请求内容不能为空"); + return AliyunMultiModalEmbedRequest.create(chatModelVo.getModelName(), contents); + } + + /** + * 执行 HTTP 请求并解析响应 + * @param request 请求对象 + * @param chatModelVo 模型配置信息 + * @return API响应对象 + * @throws IOException IO异常 + */ + private AliyunMultiModalEmbedResponse executeRequest(AliyunMultiModalEmbedRequest request, ChatModelVo chatModelVo) throws IOException { + String jsonBody = request.toJson(); + RequestBody body = RequestBody.create(jsonBody, MediaType.get("application/json")); + + Request httpRequest = new Request.Builder() + .url(chatModelVo.getApiHost()) + .addHeader("Authorization", "Bearer " + chatModelVo.getApiKey()) + .post(body) + .build(); + + try (okhttp3.Response response = okHttpClient.newCall(httpRequest).execute()) { + if (!response.isSuccessful()) { + String err = response.body() != null ? response.body().string() : "无错误信息"; + throw new IllegalArgumentException("API调用失败: " + response.code() + " - " + err, null); + } + + ResponseBody responseBody = response.body(); + if (responseBody == null) throw new IllegalArgumentException("响应体为空", null); + + return parseEmbeddingsFromResponse(responseBody.string()); + } + } + + /** + * 解析嵌入向量列表 + * @param responseBody API响应的JSON字符串 + * @return 嵌入向量响应对象 + * @throws IOException IO异常 + */ + private AliyunMultiModalEmbedResponse parseEmbeddingsFromResponse(String responseBody) throws IOException { + ObjectMapper objectMapper1 = new ObjectMapper(); + return objectMapper1.readValue(responseBody, AliyunMultiModalEmbedResponse.class); + } + + /** + * 构建 API 请求内容 Map + * @param input 多模态输入对象 + * @return 包含各种模态内容的Map列表 + */ + private List> buildContentMap(MultiModalInput input) { + List> contents = new ArrayList<>(); + + if (input.getText() != null && !input.getText().isBlank()) { + contents.add(Map.of("text", input.getText())); + } + if (input.getImageUrl() != null && !input.getImageUrl().isBlank()) { + contents.add(Map.of("image", input.getImageUrl())); + } + if (input.getVideoUrl() != null && !input.getVideoUrl().isBlank()) { + contents.add(Map.of("video", input.getVideoUrl())); + } + if (input.getMultiImageUrls() != null && input.getMultiImageUrls().length > 0) { + contents.add(Map.of("multi_images", Arrays.asList(input.getMultiImageUrls()))); + } + + return contents; + } + + /** + * 将 API 原始响应解析为 LangChain4j 的 Response + * @param resp API原始响应对象 + * @return 包含嵌入向量和token使用情况的Response对象 + */ + private Response> toEmbeddings(AliyunMultiModalEmbedResponse resp) { + if (resp == null || resp.output() == null || resp.output().embeddings() == null) { + return Response.from(Collections.emptyList()); + } + + // 转换 double -> float + List embeddings = resp.output().embeddings().stream() + .map(item -> { + float[] vector = new float[item.embedding().size()]; + for (int i = 0; i < item.embedding().size(); i++) { + vector[i] = item.embedding().get(i).floatValue(); + } + return Embedding.from(vector); + }) + .toList(); + + // 构建 TokenUsage + TokenUsage tokenUsage = null; + if (resp.usage() != null) { + tokenUsage = new TokenUsage( + resp.usage().input_tokens(), + resp.usage().image_tokens(), + resp.usage().input_tokens() +resp.usage().image_tokens() + ); + } + + return Response.from(embeddings, tokenUsage); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OllamaEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OllamaEmbeddingProvider.java new file mode 100644 index 00000000..17ed798c --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OllamaEmbeddingProvider.java @@ -0,0 +1,41 @@ +package org.ruoyi.embedding.impl; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.ollama.OllamaEmbeddingModel; +import dev.langchain4j.model.output.Response; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.BaseEmbedModelService; +import org.ruoyi.embedding.model.ModalityType; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Set; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午3:00 + * @Description: Ollama嵌入模型 + */ +@Component("ollama") +public class OllamaEmbeddingProvider implements BaseEmbedModelService { + private ChatModelVo chatModelVo; + + @Override + public void configure(ChatModelVo config) { + this.chatModelVo = config; + } + @Override + public Set getSupportedModalities() { + return Set.of(ModalityType.TEXT); + } + + @Override + public Response> embedAll(List textSegments) { + return OllamaEmbeddingModel.builder() + .baseUrl(chatModelVo.getApiHost()) + .modelName(chatModelVo.getModelName()) + .build() + .embedAll(textSegments); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OpenAiEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OpenAiEmbeddingProvider.java new file mode 100644 index 00000000..e58bfe46 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OpenAiEmbeddingProvider.java @@ -0,0 +1,43 @@ +package org.ruoyi.embedding.impl; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.openai.OpenAiEmbeddingModel; +import dev.langchain4j.model.output.Response; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.BaseEmbedModelService; +import org.ruoyi.embedding.model.ModalityType; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Set; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午3:59 + * @Description: OpenAi嵌入模型 + */ +@Component("openai") +public class OpenAiEmbeddingProvider implements BaseEmbedModelService { + protected ChatModelVo chatModelVo; + + @Override + public void configure(ChatModelVo config) { + this.chatModelVo = config; + } + + @Override + public Set getSupportedModalities() { + return Set.of(ModalityType.TEXT); + } + + @Override + public Response> embedAll(List textSegments) { + return OpenAiEmbeddingModel.builder() + .baseUrl(chatModelVo.getApiHost()) + .apiKey(chatModelVo.getApiKey()) + .modelName(chatModelVo.getModelName()) + .build() + .embedAll(textSegments); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/SiliconFlowEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/SiliconFlowEmbeddingProvider.java new file mode 100644 index 00000000..e0ccd2b9 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/SiliconFlowEmbeddingProvider.java @@ -0,0 +1,18 @@ +package org.ruoyi.embedding.impl; + + +import org.ruoyi.embedding.BaseEmbedModelService; +import org.ruoyi.embedding.model.ModalityType; +import org.springframework.stereotype.Component; + +import java.util.Set; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午3:59 + * @Description: 硅基流动（兼容 OpenAi） + */ +@Component("siliconflow") +public class SiliconFlowEmbeddingProvider extends OpenAiEmbeddingProvider { + +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/ZhiPuAiEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/ZhiPuAiEmbeddingProvider.java new file mode 100644 index 00000000..e221749a --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/ZhiPuAiEmbeddingProvider.java @@ -0,0 +1,43 @@ +package org.ruoyi.embedding.impl; + +import dev.langchain4j.community.model.zhipu.ZhipuAiEmbeddingModel; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.output.Response; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.BaseEmbedModelService; +import org.ruoyi.embedding.model.ModalityType; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Set; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午4:02 + * @Description: 智谱AI + */ +@Component("zhipu") +public class ZhiPuAiEmbeddingProvider implements BaseEmbedModelService { + private ChatModelVo chatModelVo; + + @Override + public void configure(ChatModelVo config) { + this.chatModelVo = config; + } + + @Override + public Set getSupportedModalities() { + return Set.of(); + } + + @Override + public Response> embedAll(List textSegments) { + return ZhipuAiEmbeddingModel.builder() + .baseUrl(chatModelVo.getApiHost()) + .apiKey(chatModelVo.getApiKey()) + .model(chatModelVo.getModelName()) + .build() + .embedAll(textSegments); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedRequest.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedRequest.java new file mode 100644 index 00000000..eb880588 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedRequest.java @@ -0,0 +1,44 @@ +package org.ruoyi.embedding.model; + +import org.ruoyi.common.json.utils.JsonUtils; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * @Author: Robust_H + * @Date: 2025-10-1-上午10:00 + * @Description: 阿里云多模态嵌入请求 + */ +@Data +public class AliyunMultiModalEmbedRequest { + private String model; + private Input input; + + /** + * 表示输入数据的记录类(Record) + * 该类用于封装一个包含多个映射关系列表的输入数据结构 + * + * @param contents 包含多个Map的列表，每个Map中存储String类型的键和Object类型的值 + */ + public record Input(List> contents) { } + + /** + * 创建请求对象 + */ + public static AliyunMultiModalEmbedRequest create(String modelName, List> contents) { + AliyunMultiModalEmbedRequest request = new AliyunMultiModalEmbedRequest(); + request.setModel(modelName); + Input input = new Input(contents); + request.setInput(input); + return request; + } + + /** + * 转换为JSON字符串 + */ + public String toJson() { + return JsonUtils.toJsonString(this); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedResponse.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedResponse.java new file mode 100644 index 00000000..03446d47 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedResponse.java @@ -0,0 +1,44 @@ +package org.ruoyi.embedding.model; + +import java.util.List; + +/** + * 阿里云多模态嵌入 API 响应数据模型 + */ +public record AliyunMultiModalEmbedResponse( + Output output, // 输出结果对象 + String request_id, // 请求唯一标识 + String code, // 错误码 + String message, // 错误消息 + Usage usage // 用量信息 +) { + + /** + * 输出对象，包含嵌入向量结果 + */ + public record Output( + List embeddings // 嵌入向量列表 + ) { + } + + /** + * 单个嵌入向量条目 + */ + public record EmbeddingItem( + int index, // 输入内容的索引 + List embedding, // 生成的 1024 维向量 + String type // 输入的类型 (text/image/video/multi_images) + ) { + } + + /** + * 用量统计信息 + */ + public record Usage( + int input_tokens, // 本次请求输入的 Token 数量 + int image_tokens, // 本次请求输入的图像 Token 数量 + int image_count, // 本次请求输入的图像数量 + int duration // 本次请求输入的视频时长（秒） + ) { + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/ModalityType.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/ModalityType.java new file mode 100644 index 00000000..782aac62 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/ModalityType.java @@ -0,0 +1,8 @@ +package org.ruoyi.embedding.model; + +/** + * 模态类型 + */ +public enum ModalityType { + TEXT, IMAGE, AUDIO, VIDEO, MULTI +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/MultiModalInput.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/MultiModalInput.java new file mode 100644 index 00000000..f2a31e98 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/MultiModalInput.java @@ -0,0 +1,71 @@ +package org.ruoyi.embedding.model; + +import cn.hutool.core.util.ArrayUtil; +import cn.hutool.core.util.StrUtil; +import lombok.Builder; +import lombok.Data; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午2:13 + * @Description: 多模态输入 + */ +@Data +@Builder +public class MultiModalInput { + private String text; + private byte[] imageData; + private byte[] videoData; + private String imageMimeType; + private String videoMimeType; + private String[] multiImageUrls; + private String imageUrl; + private String videoUrl; + + /** + * 检查是否有文本内容 + */ + public boolean hasText() { + return StrUtil.isNotBlank(text); + } + + /** + * 检查是否有图片内容 + */ + public boolean hasImage() { + return ArrayUtil.isNotEmpty(imageData) || StrUtil.isNotBlank(imageUrl); + } + + /** + * 检查是否有视频内容 + */ + public boolean hasVideo() { + return ArrayUtil.isNotEmpty(videoData) || StrUtil.isNotBlank(videoUrl); + } + + /** + * 检查是否有多图片 + */ + public boolean hasMultiImages() { + return ArrayUtil.isNotEmpty(multiImageUrls); + } + + /** + * 检查是否有任何内容 + */ + public boolean hasAnyContent() { + return hasText() || hasImage() || hasVideo() || hasMultiImages(); + } + + /** + * 获取内容的数量 + */ + public int getContentCount() { + int count = 0; + if (hasText()) count++; + if (hasImage()) count++; + if (hasVideo()) count++; + if (hasMultiImages()) count++; + return count; + } +} \ No newline at end of file