diff --git a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/ChatModel.java b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/ChatModel.java index e8f0e308..34465613 100644 --- a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/ChatModel.java +++ b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/ChatModel.java @@ -1,6 +1,7 @@ package org.ruoyi.domain; +import com.alibaba.excel.annotation.ExcelProperty; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; @@ -81,6 +82,11 @@ public class ChatModel extends BaseEntity { */ private Integer priority; + /** + * 模型供应商 + */ + private String ProviderName; + /** * 备注 */ diff --git a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/bo/ChatModelBo.java b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/bo/ChatModelBo.java index b828515b..34ee975e 100644 --- a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/bo/ChatModelBo.java +++ b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/bo/ChatModelBo.java @@ -1,5 +1,6 @@ package org.ruoyi.domain.bo; +import com.alibaba.excel.annotation.ExcelProperty; import io.github.linpeilie.annotations.AutoMapper; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; @@ -85,6 +86,10 @@ public class ChatModelBo extends BaseEntity { @NotBlank(message = "密钥不能为空", groups = { AddGroup.class, EditGroup.class }) private String apiKey; + /** + * 模型供应商 + */ + private String ProviderName; /** * 备注 diff --git a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/vo/ChatModelVo.java b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/vo/ChatModelVo.java index 0638c13a..6a2de3cf 100644 --- a/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/vo/ChatModelVo.java +++ b/ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/vo/ChatModelVo.java @@ -95,6 +95,12 @@ public class ChatModelVo implements Serializable { @ExcelProperty(value = "优先级") private Integer priority; + /** + * 模型供应商 + */ + @ExcelProperty(value = "模型供应商") + private String ProviderName; + /** * 备注 */ diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml b/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml index f8082a67..0394b6d1 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml +++ b/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml @@ -106,6 +106,10 @@ dashscope-sdk-java 2.19.0 + + org.ruoyi + ruoyi-chat-api + diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/KnowledgeInfo.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/KnowledgeInfo.java index 627cd3c6..f892b3c1 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/KnowledgeInfo.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/KnowledgeInfo.java @@ -83,6 +83,11 @@ public class KnowledgeInfo extends BaseEntity { */ private String vectorModelName; + /** + * 向量化模型id + */ + private Long embeddingModelId; + /** * 向量化模型名称 */ diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/KnowledgeInfoBo.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/KnowledgeInfoBo.java index 39083871..af453492 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/KnowledgeInfoBo.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/KnowledgeInfoBo.java @@ -92,7 +92,11 @@ public class KnowledgeInfoBo extends BaseEntity { /** * 向量化模型名称 */ - @NotBlank(message = "向量模型不能为空", groups = { AddGroup.class, EditGroup.class }) + private Long embeddingModelId; + + /** + * 向量化模型名称 + */ private String embeddingModelName; diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/QueryVectorBo.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/QueryVectorBo.java index ff3a26e5..0d5b2e6e 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/QueryVectorBo.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/QueryVectorBo.java @@ -31,7 +31,12 @@ public class QueryVectorBo { private String vectorModelName; /** - * 向量化模型名称 + * 向量化模型ID + */ + private Long embeddingModelId; + + /** + * 向量化模型ID */ private String embeddingModelName; diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/StoreEmbeddingBo.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/StoreEmbeddingBo.java index e4d8c381..2b87ce05 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/StoreEmbeddingBo.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/StoreEmbeddingBo.java @@ -36,6 +36,11 @@ public class StoreEmbeddingBo { */ private String vectorModelName; + /** + * 向量化模型id + */ + private Long embeddingModelId; + /** * 向量化模型名称 */ diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/vo/KnowledgeInfoVo.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/vo/KnowledgeInfoVo.java index 2dc73d45..c907e940 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/vo/KnowledgeInfoVo.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/vo/KnowledgeInfoVo.java @@ -101,6 +101,11 @@ public class KnowledgeInfoVo implements Serializable { */ private String vectorModelName; + /** + * 向量化模型id + */ + private Long embeddingModelId; + /** * 向量化模型名称 */ diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/BaseEmbedModelService.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/BaseEmbedModelService.java new file mode 100644 index 00000000..9b3d0021 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/BaseEmbedModelService.java @@ -0,0 +1,26 @@ +package org.ruoyi.embedding; + +import dev.langchain4j.model.embedding.EmbeddingModel; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.model.ModalityType; + +import java.util.Set; + +/** + * BaseEmbedModelService 接口，扩展了 EmbeddingModel 接口 + * 该接口定义了嵌入模型服务的基本配置和功能方法 + */ +public interface BaseEmbedModelService extends EmbeddingModel { + /** + * 根据配置信息配置嵌入模型 + * @param config 包含模型配置信息的 ChatModelVo 对象 + */ + void configure(ChatModelVo config); + + /** + * 获取当前嵌入模型支持的所有模态类型 + * @return 返回支持的模态类型集合 + */ + Set getSupportedModalities(); + +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/EmbeddingModelFactory.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/EmbeddingModelFactory.java new file mode 100644 index 00000000..cf0c7b60 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/EmbeddingModelFactory.java @@ -0,0 +1,117 @@ +package org.ruoyi.embedding; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.service.IChatModelService; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 嵌入模型工厂服务类 + * 负责创建和管理各种嵌入模型实例 + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class EmbeddingModelFactory { + + private final ApplicationContext applicationContext; + + private final IChatModelService chatModelService; + + // 模型缓存，使用ConcurrentHashMap保证线程安全 + private final Map modelCache = new ConcurrentHashMap<>(); + + /** + * 创建嵌入模型实例 + * 如果模型已存在于缓存中，则直接返回；否则创建新的实例 + * + * @param embeddingModelId 嵌入模型的唯一标识ID + * @return BaseEmbedModelService 嵌入模型服务实例 + */ + public BaseEmbedModelService createModel(Long embeddingModelId) { + return modelCache.computeIfAbsent(embeddingModelId, id -> { + ChatModelVo modelConfig = chatModelService.queryById(id); + if (modelConfig == null) { + throw new IllegalArgumentException("未找到模型配置，ID=" + id); + } + return createModelInstance(modelConfig.getProviderName(), modelConfig); + }); + } + + /** + * 检查模型是否支持多模态 + * + * @param embeddingModelId 嵌入模型的唯一标识ID + * @return boolean 如果模型支持多模态则返回true，否则返回false + */ + public boolean isMultimodalModel(Long embeddingModelId) { + return createModel(embeddingModelId) instanceof MultiModalEmbedModelService; + } + + /** + * 创建多模态嵌入模型实例 + * + * @param tenantId 租户ID + * @return MultiModalEmbedModelService 多模态嵌入模型服务实例 + * @throws IllegalArgumentException 当模型不支持多模态时抛出 + */ + public MultiModalEmbedModelService createMultimodalModel(Long tenantId) { + BaseEmbedModelService model = createModel(tenantId); + if (model instanceof MultiModalEmbedModelService) { + return (MultiModalEmbedModelService) model; + } + throw new IllegalArgumentException("该模型不支持多模态"); + } + + /** + * 刷新模型缓存 + * 根据给定的嵌入模型ID从缓存中移除对应的模型 + * + * @param embeddingModelId 嵌入模型的唯一标识ID + */ + public void refreshModel(Long embeddingModelId) { + // 从模型缓存中移除指定ID的模型 + modelCache.remove(embeddingModelId); + } + + /** + * 获取所有支持模型工厂的列表 + * + * @return List 支持的模型工厂名称列表 + */ + public List getSupportedFactories() { + return new ArrayList<>(applicationContext.getBeansOfType(BaseEmbedModelService.class) + .keySet()); + } + + /** + * 创建具体的模型实例 + * 根据提供的工厂名称和配置信息创建并配置模型实例 + * + * @param factory 工厂名称，用于标识模型类型 + * @param config 模型配置信息 + * @return BaseEmbedModelService 配置好的模型实例 + * @throws IllegalArgumentException 当无法获取指定的模型实例时抛出 + */ + private BaseEmbedModelService createModelInstance(String factory, ChatModelVo config) { + try { + // 从Spring上下文中获取模型实例 + BaseEmbedModelService model = applicationContext.getBean(factory, BaseEmbedModelService.class); + // 配置模型参数 + model.configure(config); + log.info("成功创建嵌入模型: factory={}, modelId={}", config.getProviderName(), config.getId()); + + return model; + } catch (NoSuchBeanDefinitionException e) { + throw new IllegalArgumentException("获取不到嵌入模型: " + factory, e); + } + } +} \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/MultiModalEmbedModelService.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/MultiModalEmbedModelService.java new file mode 100644 index 00000000..062ff00f --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/MultiModalEmbedModelService.java @@ -0,0 +1,35 @@ +package org.ruoyi.embedding; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.model.output.Response; +import org.ruoyi.embedding.model.MultiModalInput; + + +/** + * 多模态嵌入模型服务接口，继承自基础嵌入模型服务 + * 该接口提供了处理图像、视频以及多模态数据并转换为嵌入向量的功能 + */ +public interface MultiModalEmbedModelService extends BaseEmbedModelService { + /** + * 将图像数据转换为嵌入向量 + * @param imageDataUrl 图像的地址，必须是公开可访问的URL + * @return 包含嵌入向量的响应对象，可能包含状态信息和嵌入结果 + */ + Response embedImage(String imageDataUrl); + + /** + * 将视频数据转换为嵌入向量 + * @param videoDataUrl 视频的地址，必须是公开可访问的URL + * @return 包含嵌入向量的响应对象，可能包含状态信息和嵌入结果 + */ + Response embedVideo(String videoDataUrl); + + + /** + * 处理多模态输入并返回嵌入向量的方法 + * + * @param input 包含多种模态信息（如图像、文本等）的输入对象 + * @return Response 包含嵌入向量的响应对象，Embedding通常表示输入数据的向量表示 + */ + Response embedMultiModal(MultiModalInput input); +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianBaseEmbedProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianBaseEmbedProvider.java new file mode 100644 index 00000000..1511a0fe --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianBaseEmbedProvider.java @@ -0,0 +1,14 @@ +package org.ruoyi.embedding.impl; + + +import org.springframework.stereotype.Component; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午3:00 + * @Description: 阿里百炼基础嵌入模型（兼容openai） + */ +@Component("alibailian") +public class AliBaiLianBaseEmbedProvider extends OpenAiEmbeddingProvider{ + +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianMultiEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianMultiEmbeddingProvider.java new file mode 100644 index 00000000..ad3e8374 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/AliBaiLianMultiEmbeddingProvider.java @@ -0,0 +1,281 @@ +package org.ruoyi.embedding.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.output.Response; +import dev.langchain4j.model.output.TokenUsage; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.MultiModalEmbedModelService; +import org.ruoyi.embedding.model.AliyunMultiModalEmbedRequest; +import org.ruoyi.embedding.model.AliyunMultiModalEmbedResponse; +import org.ruoyi.embedding.model.ModalityType; +import org.ruoyi.embedding.model.MultiModalInput; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * 阿里云百炼多模态嵌入模型服务实现类 + * 实现了MultiModalEmbedModelService接口，提供文本、图像和视频的嵌入向量生成服务 + */ +@Component("bailianMultiModel") +@Slf4j +public class AliBaiLianMultiEmbeddingProvider implements MultiModalEmbedModelService { + private ChatModelVo chatModelVo; + + private final OkHttpClient okHttpClient; + + /** + * 构造函数，初始化HTTP客户端 + * 设置连接超时、读取超时和写入超时时间 + */ + public AliBaiLianMultiEmbeddingProvider() { + this.okHttpClient = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build(); + } + + /** + * 图像嵌入向量生成 + * @param imageDataUrl 图像数据的URL + * @return 包含图像嵌入向量的Response对象 + */ + @Override + public Response embedImage(String imageDataUrl) { + return embedSingleModality("image", imageDataUrl); + } + + /** + * 视频嵌入向量生成 + * @param videoDataUrl 视频数据的URL + * @return 包含视频嵌入向量的Response对象 + */ + @Override + public Response embedVideo(String videoDataUrl) { + return embedSingleModality("video", videoDataUrl); + } + + /** + * 多模态嵌入向量生成 + * 支持同时处理文本、图像和视频等多种模态的数据 + * @param input 包含多种模态输入的对象 + * @return 包含多模态嵌入向量的Response对象 + */ + @Override + public Response embedMultiModal(MultiModalInput input) { + try { + // 构建请求内容 + List> contents = buildContentMap(input); + if (contents.isEmpty()) { + throw new IllegalArgumentException("至少提供一种模态的内容"); + } + + // 构建请求 + AliyunMultiModalEmbedRequest request = buildRequest(contents, chatModelVo); + AliyunMultiModalEmbedResponse resp = executeRequest(request, chatModelVo); + + // 转换为 embeddings + Response> response = toEmbeddings(resp); + List embeddings = response.content(); + + if (embeddings.isEmpty()) { + log.warn("阿里云混合模态嵌入返回为空"); + return Response.from(Embedding.from(new float[0]), response.tokenUsage()); + } + + // 多模态通常取第一个向量作为代表，也可以根据业务场景返回多个 + return Response.from(embeddings.get(0), response.tokenUsage()); + + } catch (Exception e) { + log.error("阿里云混合模态嵌入失败", e); + throw new IllegalArgumentException("阿里云混合模态嵌入失败", e); + } + } + + /** + * 配置模型参数 + * @param config 模型配置信息 + */ + @Override + public void configure(ChatModelVo config) { + this.chatModelVo = config; + } + + /** + * 获取支持的模态类型 + * @return 支持的模态类型集合 + */ + @Override + public Set getSupportedModalities() { + return Set.of(ModalityType.TEXT, ModalityType.VIDEO, ModalityType.IMAGE); + } + + /** + * 批量文本嵌入向量生成 + * @param textSegments 文本段列表 + * @return 包含所有文本嵌入向量的Response对象 + */ + @Override + public Response> embedAll(List textSegments) { + if (textSegments.isEmpty()) return Response.from(Collections.emptyList()); + + try { + List> contents = new ArrayList<>(); + for (TextSegment segment : textSegments) { + contents.add(Map.of("text", segment.text())); + } + + AliyunMultiModalEmbedRequest request = buildRequest(contents, chatModelVo); + AliyunMultiModalEmbedResponse resp = executeRequest(request, chatModelVo); + + return toEmbeddings(resp); + } catch (Exception e) { + log.error("阿里云文本嵌入失败", e); + throw new IllegalArgumentException("阿里云文本嵌入失败", e); + } + } + + /** + * 单模态嵌入（图片/视频/单条文本）复用方法 + * @param key 模态类型（image/video/text） + * @param dataUrl 数据URL + * @return 包含嵌入向量的Response对象 + */ + + public Response embedSingleModality(String key, String dataUrl) { + try { + AliyunMultiModalEmbedRequest request = buildRequest(List.of(Map.of(key, dataUrl)), chatModelVo); + AliyunMultiModalEmbedResponse resp = executeRequest(request, chatModelVo); + + Response> response = toEmbeddings(resp); + List embeddings = response.content(); + + if (embeddings.isEmpty()) { + log.warn("阿里云 {} 嵌入返回为空", key); + return Response.from(Embedding.from(new float[0]), response.tokenUsage()); + } + + return Response.from(embeddings.get(0), response.tokenUsage()); + } catch (Exception e) { + log.error("阿里云 {} 嵌入失败", key, e); + throw new IllegalArgumentException("阿里云 " + key + " 嵌入失败", e); + } + } + + /** + * 构建请求对象 + * @param contents 请求内容列表 + * @param chatModelVo 模型配置信息 + * @return 构建好的请求对象 + */ + private AliyunMultiModalEmbedRequest buildRequest(List> contents, ChatModelVo chatModelVo) { + if (contents.isEmpty()) throw new IllegalArgumentException("请求内容不能为空"); + return AliyunMultiModalEmbedRequest.create(chatModelVo.getModelName(), contents); + } + + /** + * 执行 HTTP 请求并解析响应 + * @param request 请求对象 + * @param chatModelVo 模型配置信息 + * @return API响应对象 + * @throws IOException IO异常 + */ + private AliyunMultiModalEmbedResponse executeRequest(AliyunMultiModalEmbedRequest request, ChatModelVo chatModelVo) throws IOException { + String jsonBody = request.toJson(); + RequestBody body = RequestBody.create(jsonBody, MediaType.get("application/json")); + + Request httpRequest = new Request.Builder() + .url(chatModelVo.getApiHost()) + .addHeader("Authorization", "Bearer " + chatModelVo.getApiKey()) + .post(body) + .build(); + + try (okhttp3.Response response = okHttpClient.newCall(httpRequest).execute()) { + if (!response.isSuccessful()) { + String err = response.body() != null ? response.body().string() : "无错误信息"; + throw new IllegalArgumentException("API调用失败: " + response.code() + " - " + err, null); + } + + ResponseBody responseBody = response.body(); + if (responseBody == null) throw new IllegalArgumentException("响应体为空", null); + + return parseEmbeddingsFromResponse(responseBody.string()); + } + } + + /** + * 解析嵌入向量列表 + * @param responseBody API响应的JSON字符串 + * @return 嵌入向量响应对象 + * @throws IOException IO异常 + */ + private AliyunMultiModalEmbedResponse parseEmbeddingsFromResponse(String responseBody) throws IOException { + ObjectMapper objectMapper1 = new ObjectMapper(); + return objectMapper1.readValue(responseBody, AliyunMultiModalEmbedResponse.class); + } + + /** + * 构建 API 请求内容 Map + * @param input 多模态输入对象 + * @return 包含各种模态内容的Map列表 + */ + private List> buildContentMap(MultiModalInput input) { + List> contents = new ArrayList<>(); + + if (input.getText() != null && !input.getText().isBlank()) { + contents.add(Map.of("text", input.getText())); + } + if (input.getImageUrl() != null && !input.getImageUrl().isBlank()) { + contents.add(Map.of("image", input.getImageUrl())); + } + if (input.getVideoUrl() != null && !input.getVideoUrl().isBlank()) { + contents.add(Map.of("video", input.getVideoUrl())); + } + if (input.getMultiImageUrls() != null && input.getMultiImageUrls().length > 0) { + contents.add(Map.of("multi_images", Arrays.asList(input.getMultiImageUrls()))); + } + + return contents; + } + + /** + * 将 API 原始响应解析为 LangChain4j 的 Response + * @param resp API原始响应对象 + * @return 包含嵌入向量和token使用情况的Response对象 + */ + private Response> toEmbeddings(AliyunMultiModalEmbedResponse resp) { + if (resp == null || resp.output() == null || resp.output().embeddings() == null) { + return Response.from(Collections.emptyList()); + } + + // 转换 double -> float + List embeddings = resp.output().embeddings().stream() + .map(item -> { + float[] vector = new float[item.embedding().size()]; + for (int i = 0; i < item.embedding().size(); i++) { + vector[i] = item.embedding().get(i).floatValue(); + } + return Embedding.from(vector); + }) + .toList(); + + // 构建 TokenUsage + TokenUsage tokenUsage = null; + if (resp.usage() != null) { + tokenUsage = new TokenUsage( + resp.usage().input_tokens(), + resp.usage().image_tokens(), + resp.usage().input_tokens() +resp.usage().image_tokens() + ); + } + + return Response.from(embeddings, tokenUsage); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OllamaEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OllamaEmbeddingProvider.java new file mode 100644 index 00000000..17ed798c --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OllamaEmbeddingProvider.java @@ -0,0 +1,41 @@ +package org.ruoyi.embedding.impl; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.ollama.OllamaEmbeddingModel; +import dev.langchain4j.model.output.Response; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.BaseEmbedModelService; +import org.ruoyi.embedding.model.ModalityType; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Set; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午3:00 + * @Description: Ollama嵌入模型 + */ +@Component("ollama") +public class OllamaEmbeddingProvider implements BaseEmbedModelService { + private ChatModelVo chatModelVo; + + @Override + public void configure(ChatModelVo config) { + this.chatModelVo = config; + } + @Override + public Set getSupportedModalities() { + return Set.of(ModalityType.TEXT); + } + + @Override + public Response> embedAll(List textSegments) { + return OllamaEmbeddingModel.builder() + .baseUrl(chatModelVo.getApiHost()) + .modelName(chatModelVo.getModelName()) + .build() + .embedAll(textSegments); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OpenAiEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OpenAiEmbeddingProvider.java new file mode 100644 index 00000000..e58bfe46 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OpenAiEmbeddingProvider.java @@ -0,0 +1,43 @@ +package org.ruoyi.embedding.impl; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.openai.OpenAiEmbeddingModel; +import dev.langchain4j.model.output.Response; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.BaseEmbedModelService; +import org.ruoyi.embedding.model.ModalityType; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Set; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午3:59 + * @Description: OpenAi嵌入模型 + */ +@Component("openai") +public class OpenAiEmbeddingProvider implements BaseEmbedModelService { + protected ChatModelVo chatModelVo; + + @Override + public void configure(ChatModelVo config) { + this.chatModelVo = config; + } + + @Override + public Set getSupportedModalities() { + return Set.of(ModalityType.TEXT); + } + + @Override + public Response> embedAll(List textSegments) { + return OpenAiEmbeddingModel.builder() + .baseUrl(chatModelVo.getApiHost()) + .apiKey(chatModelVo.getApiKey()) + .modelName(chatModelVo.getModelName()) + .build() + .embedAll(textSegments); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/SiliconFlowEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/SiliconFlowEmbeddingProvider.java new file mode 100644 index 00000000..e0ccd2b9 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/SiliconFlowEmbeddingProvider.java @@ -0,0 +1,18 @@ +package org.ruoyi.embedding.impl; + + +import org.ruoyi.embedding.BaseEmbedModelService; +import org.ruoyi.embedding.model.ModalityType; +import org.springframework.stereotype.Component; + +import java.util.Set; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午3:59 + * @Description: 硅基流动（兼容 OpenAi） + */ +@Component("siliconflow") +public class SiliconFlowEmbeddingProvider extends OpenAiEmbeddingProvider { + +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/ZhiPuAiEmbeddingProvider.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/ZhiPuAiEmbeddingProvider.java new file mode 100644 index 00000000..e221749a --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/ZhiPuAiEmbeddingProvider.java @@ -0,0 +1,43 @@ +package org.ruoyi.embedding.impl; + +import dev.langchain4j.community.model.zhipu.ZhipuAiEmbeddingModel; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.output.Response; +import org.ruoyi.domain.vo.ChatModelVo; +import org.ruoyi.embedding.BaseEmbedModelService; +import org.ruoyi.embedding.model.ModalityType; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Set; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午4:02 + * @Description: 智谱AI + */ +@Component("zhipu") +public class ZhiPuAiEmbeddingProvider implements BaseEmbedModelService { + private ChatModelVo chatModelVo; + + @Override + public void configure(ChatModelVo config) { + this.chatModelVo = config; + } + + @Override + public Set getSupportedModalities() { + return Set.of(); + } + + @Override + public Response> embedAll(List textSegments) { + return ZhipuAiEmbeddingModel.builder() + .baseUrl(chatModelVo.getApiHost()) + .apiKey(chatModelVo.getApiKey()) + .model(chatModelVo.getModelName()) + .build() + .embedAll(textSegments); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedRequest.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedRequest.java new file mode 100644 index 00000000..eb880588 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedRequest.java @@ -0,0 +1,44 @@ +package org.ruoyi.embedding.model; + +import org.ruoyi.common.json.utils.JsonUtils; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * @Author: Robust_H + * @Date: 2025-10-1-上午10:00 + * @Description: 阿里云多模态嵌入请求 + */ +@Data +public class AliyunMultiModalEmbedRequest { + private String model; + private Input input; + + /** + * 表示输入数据的记录类(Record) + * 该类用于封装一个包含多个映射关系列表的输入数据结构 + * + * @param contents 包含多个Map的列表，每个Map中存储String类型的键和Object类型的值 + */ + public record Input(List> contents) { } + + /** + * 创建请求对象 + */ + public static AliyunMultiModalEmbedRequest create(String modelName, List> contents) { + AliyunMultiModalEmbedRequest request = new AliyunMultiModalEmbedRequest(); + request.setModel(modelName); + Input input = new Input(contents); + request.setInput(input); + return request; + } + + /** + * 转换为JSON字符串 + */ + public String toJson() { + return JsonUtils.toJsonString(this); + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedResponse.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedResponse.java new file mode 100644 index 00000000..03446d47 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/AliyunMultiModalEmbedResponse.java @@ -0,0 +1,44 @@ +package org.ruoyi.embedding.model; + +import java.util.List; + +/** + * 阿里云多模态嵌入 API 响应数据模型 + */ +public record AliyunMultiModalEmbedResponse( + Output output, // 输出结果对象 + String request_id, // 请求唯一标识 + String code, // 错误码 + String message, // 错误消息 + Usage usage // 用量信息 +) { + + /** + * 输出对象，包含嵌入向量结果 + */ + public record Output( + List embeddings // 嵌入向量列表 + ) { + } + + /** + * 单个嵌入向量条目 + */ + public record EmbeddingItem( + int index, // 输入内容的索引 + List embedding, // 生成的 1024 维向量 + String type // 输入的类型 (text/image/video/multi_images) + ) { + } + + /** + * 用量统计信息 + */ + public record Usage( + int input_tokens, // 本次请求输入的 Token 数量 + int image_tokens, // 本次请求输入的图像 Token 数量 + int image_count, // 本次请求输入的图像数量 + int duration // 本次请求输入的视频时长（秒） + ) { + } +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/ModalityType.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/ModalityType.java new file mode 100644 index 00000000..782aac62 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/ModalityType.java @@ -0,0 +1,8 @@ +package org.ruoyi.embedding.model; + +/** + * 模态类型 + */ +public enum ModalityType { + TEXT, IMAGE, AUDIO, VIDEO, MULTI +} diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/MultiModalInput.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/MultiModalInput.java new file mode 100644 index 00000000..f2a31e98 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/model/MultiModalInput.java @@ -0,0 +1,71 @@ +package org.ruoyi.embedding.model; + +import cn.hutool.core.util.ArrayUtil; +import cn.hutool.core.util.StrUtil; +import lombok.Builder; +import lombok.Data; + +/** + * @Author: Robust_H + * @Date: 2025-09-30-下午2:13 + * @Description: 多模态输入 + */ +@Data +@Builder +public class MultiModalInput { + private String text; + private byte[] imageData; + private byte[] videoData; + private String imageMimeType; + private String videoMimeType; + private String[] multiImageUrls; + private String imageUrl; + private String videoUrl; + + /** + * 检查是否有文本内容 + */ + public boolean hasText() { + return StrUtil.isNotBlank(text); + } + + /** + * 检查是否有图片内容 + */ + public boolean hasImage() { + return ArrayUtil.isNotEmpty(imageData) || StrUtil.isNotBlank(imageUrl); + } + + /** + * 检查是否有视频内容 + */ + public boolean hasVideo() { + return ArrayUtil.isNotEmpty(videoData) || StrUtil.isNotBlank(videoUrl); + } + + /** + * 检查是否有多图片 + */ + public boolean hasMultiImages() { + return ArrayUtil.isNotEmpty(multiImageUrls); + } + + /** + * 检查是否有任何内容 + */ + public boolean hasAnyContent() { + return hasText() || hasImage() || hasVideo() || hasMultiImages(); + } + + /** + * 获取内容的数量 + */ + public int getContentCount() { + int count = 0; + if (hasText()) count++; + if (hasImage()) count++; + if (hasVideo()) count++; + if (hasMultiImages()) count++; + return count; + } +} \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java index 799ce729..d81905ae 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java @@ -28,6 +28,8 @@ import lombok.extern.slf4j.Slf4j; import org.ruoyi.common.core.service.ConfigService; import org.ruoyi.domain.bo.QueryVectorBo; import org.ruoyi.domain.bo.StoreEmbeddingBo; +import org.ruoyi.embedding.BaseEmbedModelService; +import org.ruoyi.embedding.EmbeddingModelFactory; import org.ruoyi.service.VectorStoreService; import org.springframework.stereotype.Service; import java.util.*; @@ -48,6 +50,8 @@ public class VectorStoreServiceImpl implements VectorStoreService { // private EmbeddingStore embeddingStore; private WeaviateClient client; + private final EmbeddingModelFactory embeddingModelFactory; + @Override public void createSchema(String kid, String modelName) { @@ -98,18 +102,16 @@ public class VectorStoreServiceImpl implements VectorStoreService { @Override public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) { createSchema(storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName()); - EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), - storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl()); + BaseEmbedModelService model = embeddingModelFactory.createModel(storeEmbeddingBo.getEmbeddingModelId()); List chunkList = storeEmbeddingBo.getChunkList(); List fidList = storeEmbeddingBo.getFids(); String kid = storeEmbeddingBo.getKid(); String docId = storeEmbeddingBo.getDocId(); - log.info("向量存储条数记录: " + chunkList.size()); long startTime = System.currentTimeMillis(); for (int i = 0; i < chunkList.size(); i++) { String text = chunkList.get(i); String fid = fidList.get(i); - Embedding embedding = embeddingModel.embed(text).content(); + Embedding embedding = model.embed(text).content(); Map properties = Map.of( "text", text, "fid",fid, @@ -137,9 +139,8 @@ public class VectorStoreServiceImpl implements VectorStoreService { @Override public List getQueryVector(QueryVectorBo queryVectorBo) { createSchema(queryVectorBo.getKid(), queryVectorBo.getVectorModelName()); - EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(), - queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl()); - Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content(); + BaseEmbedModelService model = embeddingModelFactory.createModel(queryVectorBo.getEmbeddingModelId()); + Embedding queryEmbedding = model.embed(queryVectorBo.getQuery()).content(); float[] vector = queryEmbedding.vector(); List vectorStrings = new ArrayList<>(); for (float v : vector) { @@ -246,28 +247,4 @@ public class VectorStoreServiceImpl implements VectorStoreService { log.error("删除失败: {}", result.getError()); } } - - /** - * 获取向量模型 - */ - @SneakyThrows - public EmbeddingModel getEmbeddingModel(String modelName, String apiKey, String baseUrl) { - EmbeddingModel embeddingModel; - if ("quentinz/bge-large-zh-v1.5".equals(modelName)) { - embeddingModel = OllamaEmbeddingModel.builder() - .baseUrl(baseUrl) - .modelName(modelName) - .build(); - } else if ("baai/bge-m3".equals(modelName)) { - embeddingModel = OpenAiEmbeddingModel.builder() - .apiKey(apiKey) - .baseUrl(baseUrl) - .modelName(modelName) - .build(); - } else { - throw new ServiceException("未找到对应向量化模型!"); - } - return embeddingModel; - } - } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/KnowledgeInfoServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/KnowledgeInfoServiceImpl.java index 089a0ee8..3b88de9b 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/KnowledgeInfoServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/KnowledgeInfoServiceImpl.java @@ -290,7 +290,7 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { .eq(KnowledgeInfo::getId, kid)); // 通过向量模型查询模型信息 - ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModelName()); + ChatModelVo chatModelVo = chatModelService.queryById(knowledgeInfoVo.getEmbeddingModelId()); StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo(); storeEmbeddingBo.setKid(kid); @@ -298,7 +298,7 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { storeEmbeddingBo.setFids(fids); storeEmbeddingBo.setChunkList(chunkList); storeEmbeddingBo.setVectorModelName(knowledgeInfoVo.getVectorModelName()); - storeEmbeddingBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModelName()); + storeEmbeddingBo.setEmbeddingModelId(knowledgeInfoVo.getEmbeddingModelId()); storeEmbeddingBo.setApiKey(chatModelVo.getApiKey()); storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost()); vectorStoreService.storeEmbeddings(storeEmbeddingBo); diff --git a/script/sql/update/2025-10-4-多供应商嵌入模型集成.sql b/script/sql/update/2025-10-4-多供应商嵌入模型集成.sql new file mode 100644 index 00000000..6619c20f --- /dev/null +++ b/script/sql/update/2025-10-4-多供应商嵌入模型集成.sql @@ -0,0 +1,13 @@ +-- 为 chat_model 表添加 provider_name 字段 +-- 变更日期: 2025-10-04 +-- 负责人: Robust_H +-- 说明: 嵌入模型供应商 （用于实现动态选择嵌入模型实现类） +ALTER TABLE `ruoyi-ai`.chat_model + ADD COLUMN `provider_name` varchar(20) DEFAULT NULL COMMENT '模型供应商' AFTER `model_name`; + +-- 修改 knowledge_info 中的 ‘embedding_model_name’ 为 ‘embedding_model_id’ +-- 变更日期: 2025-10-04 +-- 负责人: Robust_H +-- 说明: 用于区分多个供应商实现同一嵌入模型的情况 +ALTER TABLE `ruoyi-ai`.knowledge_info + ADD COLUMN `embedding_model_id` bigint DEFAULT NULL COMMENT '模型id' AFTER `embedding_model_name`;