xieyinlu/ruoyi-ai
1
0
Fork 0
mirror of https://gitcode.com/ageerle/ruoyi-ai.git synced 2026-09-16 01:44:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #315 from MuSan-Li/update/2026-07-19-fixes

Browse Source 
修复RAG相关逻辑优化:统一文档切割配置、向量数据生命周期、
统一检索服务(阈值/重排/缓存)、按知识库路由向量库、重新解析支持
This commit is contained in:
ageerle@163.com
2026-07-21 14:13:42 +08:00
committed by ageerle
parent 24bee53f9e cf5423f69f
commit 46a8d6b552
68 changed files with 1311 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/knowledge/KnowledgeAttachController.java
View File

@@ -9,6 +9,7 @@ import cn.dev33.satoken.annotation.SaCheckPermission;
import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo;
import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo;
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
import org.ruoyi.service.knowledge.IKnowledgeAttachService;
import org.springframework.web.bind.annotation.*;
import org.springframework.validation.annotation.Validated;
@@ -106,7 +107,10 @@ public class KnowledgeAttachController extends BaseController {
/**
* 上传知识库附件
* 注意：multipart 上传不能加 @RepeatSubmit（其参数序列化不支持 MultipartFile）
*/
@SaCheckPermission("system:attach:add")
@Log(title = "知识库附件", businessType = BusinessType.INSERT)
@PostMapping(value = "/upload")
public R<String> upload(KnowledgeInfoUploadBo bo){
knowledgeAttachService.upload(bo);
@@ -118,9 +122,20 @@ public class KnowledgeAttachController extends BaseController {
*
* @param id 附件ID
*/
@SaCheckPermission("system:attach:edit")
@Log(title = "知识库附件", businessType = BusinessType.UPDATE)
@PostMapping("/parse/{id}")
@RepeatSubmit()
public R<Void> parse(@PathVariable Long id) {
knowledgeAttachService.parse(id);
return R.ok();
}
@SaCheckPermission("system:attach:edit")
@Log(title = "知识库附件批量重新解析", businessType = BusinessType.UPDATE)
@PostMapping("/reparse/knowledge/{knowledgeId}")
@RepeatSubmit()
public R<KnowledgeReparseVo> reparseKnowledge(@PathVariable Long knowledgeId) {
return R.ok(knowledgeAttachService.reparseKnowledge(knowledgeId));
}
}

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/knowledge/KnowledgeFragmentController.java
View File

@@ -107,7 +107,9 @@ public class KnowledgeFragmentController extends BaseController {
/**
* 检索测试
*/
@SaCheckPermission("system:fragment:list")
@PostMapping("/retrieval")
@RepeatSubmit()
public R<List<KnowledgeRetrievalVo>> retrieval(@RequestBody KnowledgeFragmentBo bo) {
return R.ok(knowledgeFragmentService.retrieval(bo));
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/knowledge/KnowledgeAttach.java
View File

@@ -37,6 +37,9 @@ public class KnowledgeAttach extends BaseEntity {
*/
private String docId;
/** SHA-256 content digest used for upload idempotency. */
private String fileHash;
/**
* 附件名称
*/

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/knowledge/KnowledgeFragment.java
View File

@@ -27,6 +27,11 @@ public class KnowledgeFragment extends BaseEntity {
@TableId(value = "id")
private Long id;
/**
* 向量库片段ID（与向量库中的 fid 元数据对应，用于向量定位与混合检索融合）
*/
private String fid;
/**
* 文档ID-用于关联文本块信息
*/

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/knowledge/KnowledgeFragmentVo.java
View File

@@ -30,6 +30,11 @@ public class KnowledgeFragmentVo implements Serializable {
@ExcelProperty(value = "主键")
private Long id;
/**
* 向量库片段ID
*/
private String fid;
/**
* 文档ID-用于关联文本块信息
*/

4
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/knowledge/KnowledgeReparseVo.java Normal file
View File

@@ -0,0 +1,4 @@
package org.ruoyi.domain.vo.knowledge;
public record KnowledgeReparseVo(int submitted, int skipped, int total) {
}

15
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/EmbeddingModelFactory.java
View File

@@ -77,13 +77,22 @@ public class EmbeddingModelFactory {
/**
* 刷新模型缓存
* 根据给定的嵌入模型ID从缓存中移除对应的模型
* 根据给定的嵌入模型ID解析模型名称后，从缓存中移除对应的模型
*
* @param embeddingModelId 嵌入模型的唯一标识ID
*/
public void refreshModel(Long embeddingModelId) {
// 从模型缓存中移除指定ID的模型
modelCache.remove(embeddingModelId);
ChatModelVo modelConfig = chatModelService.queryById(embeddingModelId);
if (modelConfig != null) {
modelCache.remove(modelConfig.getModelName());
}
}
/**
* 按模型名称刷新缓存
*/
public void refreshModelByName(String embeddingModelName) {
modelCache.remove(embeddingModelName);
}
/**

14
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/RerankModelFactory.java
View File

@@ -55,12 +55,22 @@ public class RerankModelFactory {
/**
* 刷新模型缓存
* 根据给定的模型ID从缓存中移除对应的模型
* 根据给定的模型ID解析模型名称后，从缓存中移除对应的模型
*
* @param modelId 模型的唯一标识ID
*/
public void refreshModel(Long modelId) {
modelCache.remove(modelId);
ChatModelVo modelConfig = chatModelService.queryById(modelId);
if (modelConfig != null) {
modelCache.remove(modelConfig.getModelName());
}
}
/**
* 按模型名称刷新缓存
*/
public void refreshModelByName(String modelName) {
modelCache.remove(modelName);
}
/**

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/ResourceLoaderFactory.java
View File

@@ -17,7 +17,7 @@ public class ResourceLoaderFactory {
private final ExcelTextSplitter excelTextSplitter;
public ResourceLoader getLoaderByFileType(String fileType) {
fileType = StringUtils.removeStart(fileType, ".");
fileType = StringUtils.lowerCase(StringUtils.removeStart(StringUtils.trim(fileType), "."));
if (FileTypeConstants.isTextFile(fileType)) {
return new TextFileLoader(characterTextSplitter);
} else if (FileTypeConstants.isWord(fileType)) {

13
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/VectorStoreStrategyFactory.java
View File

@@ -10,6 +10,7 @@ import org.ruoyi.service.vector.impl.MilvusVectorStoreStrategy;
import org.ruoyi.service.vector.impl.QdrantVectorStoreStrategy;
import org.ruoyi.service.vector.impl.WeaviateVectorStoreStrategy;
import org.springframework.stereotype.Component;
import org.ruoyi.common.core.exception.ServiceException;
import java.util.HashMap;
import java.util.Map;
@@ -45,14 +46,20 @@ public class VectorStoreStrategyFactory {
* 获取当前配置的向量库策略
*/
public VectorStoreService getStrategy() {
String vectorStoreType = vectorStoreProperties.getType();
return getStrategy(null);
}
public VectorStoreService getStrategy(String requestedType) {
String vectorStoreType = requestedType;
if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) {
vectorStoreType = vectorStoreProperties.getType();
}
if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) {
vectorStoreType = "weaviate"; // 默认使用weaviate
}
VectorStoreService strategy = strategies.get(vectorStoreType.toLowerCase());
if (strategy == null) {
log.warn("未找到向量库策略: {}, 使用默认策略: weaviate", vectorStoreType);
strategy = strategies.get("weaviate");
throw new ServiceException("不支持的向量库类型: " + vectorStoreType);
}
log.debug("使用向量库策略: {}", vectorStoreType);
return strategy;

8
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mapper/knowledge/KnowledgeAttachMapper.java
View File

@@ -6,6 +6,8 @@ import org.apache.ibatis.annotations.Select;
import org.ruoyi.domain.entity.knowledge.KnowledgeAttach;
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
import org.ruoyi.common.mybatis.core.mapper.BaseMapperPlus;
import java.util.List;
import java.util.Map;
/**
* 知识库附件Mapper接口
@@ -21,4 +23,10 @@ public interface KnowledgeAttachMapper extends BaseMapperPlus<KnowledgeAttach, K
*/
@Select("SELECT COUNT(*) FROM knowledge_attach WHERE knowledge_id = #{knowledgeId}")
int countByKnowledgeId(@Param("knowledgeId") Long knowledgeId);
@Select("<script>SELECT knowledge_id AS knowledgeId, COUNT(*) AS documentCount " +
"FROM knowledge_attach WHERE knowledge_id IN " +
"<foreach collection='knowledgeIds' item='id' open='(' separator=',' close=')'>#{id}</foreach> " +
"GROUP BY knowledge_id</script>")
List<Map<String, Object>> countByKnowledgeIds(@Param("knowledgeIds") List<Long> knowledgeIds);
}

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mapper/knowledge/KnowledgeFragmentMapper.java
View File

@@ -34,7 +34,7 @@ public interface KnowledgeFragmentMapper extends BaseMapperPlus<KnowledgeFragmen
"</script>")
List<DocFragmentCountVo> selectFragmentCountByDocIds(@Param("docIds") List<String> docIds);
@Select("<script>" +
"SELECT id, doc_id AS docId, content, idx, knowledge_id AS knowledgeId " +
"SELECT id, fid, doc_id AS docId, content, idx, knowledge_id AS knowledgeId " +
"FROM knowledge_fragment " +
"WHERE knowledge_id = #{knowledgeId} " +
"AND MATCH (content) AGAINST (#{query} IN NATURAL LANGUAGE MODE) " +

123
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/chat/impl/ChatServiceFacade.java
View File

@@ -77,6 +77,7 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.LinkedHashMap;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
@@ -161,7 +162,8 @@ public class ChatServiceFacade implements IChatService {
throw new IllegalArgumentException("模型不存在: " + chatRequest.getModel());
}
// 2. 构建上下文消息列表
// 2. 构建上下文消息列表（系统提示词 + 历史消息 + 当前用户消息）
// 注意：RAG 检索增强统一在 handleAgentChat 中执行一次，此处不再重复检索
List<ChatMessage> contextMessages = buildContextMessages(chatRequest, agentVo);
chatRequest.setEmitter(emitter);
@@ -282,12 +284,20 @@ public class ChatServiceFacade implements IChatService {
.responseStrategy(SupervisorResponseStrategy.SUMMARY);
SupervisorAgent supervisor = supervisorBuilder.build();
// 知识库增强：智能体绑定了知识库时，对 supervisor 输入做一次 RAG 增强
// 知识库增强：智能体绑定了知识库时，对 supervisor 输入做一次 RAG 增强（全程唯一一次检索）
String augmentedInput = augmentAgentInput(chatRequest, agentVo);
// 智能体自定义系统提示词：supervisor builder 不支持 systemMessage，前置到输入
String prompt = (agentVo != null && StringUtils.isNotBlank(agentVo.getSystemPrompt()))
? agentVo.getSystemPrompt() + "\n\n" + augmentedInput
: augmentedInput;
// 组装最终 prompt：系统提示词 → 多轮历史 → RAG 增强后的当前提问
StringBuilder promptBuilder = new StringBuilder();
if (agentVo != null && StringUtils.isNotBlank(agentVo.getSystemPrompt())) {
promptBuilder.append(agentVo.getSystemPrompt()).append("\n\n");
}
String historyText = formatHistoryMessages(chatRequest.getContextMessages(), chatRequest.getContent());
if (StringUtils.isNotBlank(historyText)) {
promptBuilder.append("以下是本次会话的历史对话，请结合上下文理解用户最新提问：\n")
.append(historyText).append("\n\n");
}
promptBuilder.append(augmentedInput);
String prompt = promptBuilder.toString();
String tokenValue = chatRequest.getTokenValue();
@@ -316,8 +326,9 @@ public class ChatServiceFacade implements IChatService {
* 兜底 MCP 工具装配（无智能体时使用，保留原有 3 个硬编码客户端逻辑）
*/
private ToolProvider buildDefaultMcpToolProvider(Long userId) {
String npxCommand = resolveNpxCommand();
McpTransport playwrightTransport = new StdioMcpTransport.Builder()
.command(List.of("C:\\Program Files\\nodejs\\npx.cmd", "-y", "@playwright/mcp@latest"))
.command(List.of(npxCommand, "-y", "@playwright/mcp@latest"))
.logEvents(true)
.build();
McpClient playwrightMcpClient = new DefaultMcpClient.Builder()
@@ -327,7 +338,7 @@ public class ChatServiceFacade implements IChatService {
String userDir = System.getProperty("user.dir");
McpTransport filesystemTransport = new StdioMcpTransport.Builder()
.command(List.of("C:\\Program Files\\nodejs\\npx.cmd", "-y",
.command(List.of(npxCommand, "-y",
"@modelcontextprotocol/server-filesystem", userDir))
.logEvents(true)
.build();
@@ -341,6 +352,14 @@ public class ChatServiceFacade implements IChatService {
.build();
}
private String resolveNpxCommand() {
String configured = System.getProperty("mcp.npx.command");
if (StringUtils.isNotBlank(configured)) return configured;
String fromEnv = System.getenv("MCP_NPX_COMMAND");
if (StringUtils.isNotBlank(fromEnv)) return fromEnv;
return System.getProperty("os.name", "").toLowerCase().contains("win") ? "npx.cmd" : "npx";
}
/**
* 装配磁盘 ShellSkills：智能体勾选了技能名时按名过滤，否则加载全部。
* 无 skills 时返回 null（调用方据此跳过 SkillsAgent 的 toolProvider 注入）
@@ -476,27 +495,7 @@ public class ChatServiceFacade implements IChatService {
messages.add(SystemMessage.from(agentVo.getSystemPrompt()));
}
// 1. 初始化当前用户消息
UserMessage userMessage = UserMessage.userMessage(chatRequest.getContent());
// 2. 知识库检索增强 (RAG)：智能体的 knowledgeIds 优先，回退到请求的 knowledgeId
List<Long> knowledgeIds = collectKnowledgeIds(chatRequest, agentVo);
if (knowledgeIds != null && !knowledgeIds.isEmpty()) {
RetrievalAugmentor augmentor = buildMultiKnowledgeAugmentor(knowledgeIds);
if (augmentor != null) {
log.info("执行多知识库 RAG 流程: kids={}", knowledgeIds);
Metadata metadata = Metadata.from(userMessage, chatRequest.getSessionId(), new ArrayList<>());
AugmentationRequest augmentationRequest = new AugmentationRequest(userMessage, metadata);
AugmentationResult result = augmentor.augment(augmentationRequest);
ChatMessage augmented = result.chatMessage();
if (augmented instanceof UserMessage) {
userMessage = (UserMessage) augmented;
log.debug("RAG 增强完成，UserMessage 已注入背景知识");
}
}
}
// 3. 从数据库查询历史对话消息（放在前面）
// 1. 从数据库查询历史对话消息（放在前面）
if (chatRequest.getSessionId() != null) {
MessageWindowChatMemory memory = createChatMemory(chatRequest.getSessionId());
if (memory != null) {
@@ -508,12 +507,37 @@ public class ChatServiceFacade implements IChatService {
}
}
// 4. 添加经过增强的用户消息（放在最后）
messages.add(userMessage);
// 2. 添加当前用户消息（放在最后；RAG 增强在 handleAgentChat 中统一执行，避免重复检索）
messages.add(UserMessage.userMessage(chatRequest.getContent()));
return messages;
}
/**
* 将上下文消息格式化为多轮对话文本（供只接受 String 输入的 Supervisor 使用）。
* 跳过 SystemMessage（系统提示词单独前置）与最后一条当前用户消息（单独做 RAG 增强后拼接）。
*/
private String formatHistoryMessages(List<ChatMessage> contextMessages, String currentContent) {
if (contextMessages == null || contextMessages.isEmpty()) {
return "";
}
StringBuilder sb = new StringBuilder();
int limit = contextMessages.size();
// 最后一条是当前用户消息，不纳入历史（避免与增强后的输入重复）
if (limit > 0 && contextMessages.get(limit - 1) instanceof UserMessage) {
limit--;
}
for (int i = 0; i < limit; i++) {
ChatMessage msg = contextMessages.get(i);
if (msg instanceof UserMessage userMsg) {
sb.append("用户: ").append(userMsg.singleText()).append("\n");
} else if (msg instanceof AiMessage aiMsg) {
sb.append("助手: ").append(aiMsg.text()).append("\n");
}
}
return sb.toString().trim();
}
/**
* 汇总本次对话要检索的知识库ID列表：智能体绑定的 knowledgeIds 优先，回退到请求的 knowledgeId
*/
@@ -580,18 +604,35 @@ public class ChatServiceFacade implements IChatService {
@Override
public List<Content> retrieve(Query query) {
List<Content> all = new ArrayList<>();
for (ContentRetriever r : delegates) {
try {
List<Content> part = r.retrieve(query);
if (part != null) {
all.addAll(part);
}
} catch (Exception e) {
log.warn("复合检索子检索器异常: {}", e.getMessage());
List<CompletableFuture<List<Content>>> futures = delegates.stream()
.map(r -> CompletableFuture.supplyAsync(() -> {
try {
List<Content> part = r.retrieve(query);
return part == null ? List.<Content>of() : part;
} catch (Exception e) {
log.warn("复合检索子检索器异常: {}", e.getMessage());
return List.<Content>of();
}
})).toList();
Map<String, Content> unique = new LinkedHashMap<>();
for (CompletableFuture<List<Content>> future : futures) {
for (Content content : future.join()) {
String key = content.textSegment().metadata().getString("kid") + "|"
+ content.textSegment().metadata().getString("docId") + "|"
+ content.textSegment().metadata().getString("fid");
if (key.endsWith("null|null|null")) key = content.textSegment().text();
unique.putIfAbsent(key, content);
}
}
return all;
List<Content> bounded = new ArrayList<>();
int chars = 0;
for (Content content : unique.values()) {
int next = content.textSegment().text().length();
if (bounded.size() >= 20 || chars + next > 24000) break;
bounded.add(content);
chars += next;
}
return bounded;
}
}

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/embed/impl/AliBaiLianBaseEmbedProvider.java
View File

@@ -19,6 +19,7 @@ import java.util.Set;
* @Description: 阿里百炼基础嵌入模型（兼容openai）
*/
@Component("alibailian")
@org.springframework.context.annotation.Scope("prototype")
public class AliBaiLianBaseEmbedProvider extends OpenAiEmbeddingProvider {
private ChatModelVo chatModelVo;

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/embed/impl/AliBaiLianMultiEmbeddingProvider.java
View File

@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
* 实现了MultiModalEmbedModelService接口，提供文本、图像和视频的嵌入向量生成服务
*/
@Component("bailianMultiModel")
@org.springframework.context.annotation.Scope("prototype")
@Slf4j
public class AliBaiLianMultiEmbeddingProvider implements MultiModalEmbedModelService {
private final OkHttpClient okHttpClient;

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/embed/impl/MinimaxEmbeddingProvider.java
View File

@@ -12,6 +12,7 @@ import org.springframework.stereotype.Component;
* @date 2026/3/21
*/
@Component("minimax")
@org.springframework.context.annotation.Scope("prototype")
public class MinimaxEmbeddingProvider extends OpenAiEmbeddingProvider {
}

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/embed/impl/OllamaEmbeddingProvider.java
View File

@@ -19,6 +19,7 @@ import java.util.Set;
* @Description: Ollama嵌入模型
*/
@Component("ollama")
@org.springframework.context.annotation.Scope("prototype")
public class OllamaEmbeddingProvider implements BaseEmbedModelService {
private ChatModelVo chatModelVo;

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/embed/impl/OpenAiEmbeddingProvider.java
View File

@@ -19,6 +19,7 @@ import java.util.Set;
* @Description: OpenAi嵌入模型
*/
@Component("openai")
@org.springframework.context.annotation.Scope("prototype")
public class OpenAiEmbeddingProvider implements BaseEmbedModelService {
protected ChatModelVo chatModelVo;

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/embed/impl/SiliconFlowEmbeddingProvider.java
View File

@@ -9,6 +9,7 @@ import org.springframework.stereotype.Component;
* @Description: 硅基流动（兼容 OpenAi）
*/
@Component("siliconflow")
@org.springframework.context.annotation.Scope("prototype")
public class SiliconFlowEmbeddingProvider extends OpenAiEmbeddingProvider {
}

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/embed/impl/ZhipuAiEmbeddingProvider.java
View File

@@ -21,6 +21,7 @@ import java.util.Set;
* @Description: 智谱AI嵌入模型
*/
@Component("zhipu")
@org.springframework.context.annotation.Scope("prototype")
public class ZhipuAiEmbeddingProvider implements BaseEmbedModelService {
protected ChatModelVo chatModelVo;

20
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/DocumentSplitConfig.java Normal file
View File

@@ -0,0 +1,20 @@
package org.ruoyi.service.knowledge;
import org.ruoyi.common.core.exception.ServiceException;
/** Immutable snapshot of the split settings used for one parse operation. */
public record DocumentSplitConfig(String separator, int blockSize, int overlap, String fileType) {
public static final int DEFAULT_BLOCK_SIZE = 1000;
public static final int DEFAULT_OVERLAP = 50;
public DocumentSplitConfig {
if (blockSize <= 0) {
throw new ServiceException("文本块大小必须大于0");
}
if (overlap < 0 || overlap >= blockSize) {
throw new ServiceException("重叠字符数必须大于等于0且小于文本块大小");
}
fileType = fileType == null ? "" : fileType.strip().replaceFirst("^\\.", "").toLowerCase();
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/IKnowledgeAttachService.java
View File

@@ -5,6 +5,7 @@ import org.ruoyi.common.mybatis.core.page.PageQuery;
import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo;
import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo;
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
import java.util.Collection;
import java.util.List;
@@ -79,4 +80,6 @@ public interface IKnowledgeAttachService {
* @param id 附件ID
*/
void parse(Long id);
KnowledgeReparseVo reparseKnowledge(Long knowledgeId);
}

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/ResourceLoader.java
View File

@@ -10,5 +10,5 @@ public interface ResourceLoader {
String getContent(InputStream inputStream);
List<String> getChunkList(String content, String kid);
List<String> getChunkList(String content, DocumentSplitConfig config);
}

4
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/TextSplitter.java
View File

@@ -11,8 +11,8 @@ public interface TextSplitter {
* 文本切分
*
* @param content 文本内容
* @param kid 知识库id
* @param config 本次解析的分片配置快照
* @return 切分后的文本列表
*/
List<String> split(String content, String kid);
List<String> split(String content, DocumentSplitConfig config);
}

130
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java
View File

@@ -2,6 +2,7 @@ package org.ruoyi.service.knowledge.impl;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@@ -11,6 +12,7 @@ import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
import org.ruoyi.common.chat.service.chat.IChatModelService;
import org.ruoyi.enums.KnowledgeAttachStatus;
import org.ruoyi.common.core.domain.dto.OssDTO;
import org.ruoyi.common.core.exception.ServiceException;
import org.ruoyi.common.core.service.OssService;
import org.ruoyi.common.core.utils.MapstructUtils;
import org.ruoyi.common.core.utils.SpringUtils;
@@ -25,13 +27,16 @@ import org.ruoyi.domain.entity.knowledge.KnowledgeFragment;
import org.ruoyi.domain.vo.knowledge.DocFragmentCountVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
import org.ruoyi.factory.ResourceLoaderFactory;
import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper;
import org.ruoyi.mapper.knowledge.KnowledgeFragmentMapper;
import org.ruoyi.service.knowledge.IKnowledgeAttachService;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.vector.VectorStoreService;
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
@@ -60,6 +65,7 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
private final ResourceLoaderFactory resourceLoaderFactory;
private final VectorStoreService vectorStoreService;
private final OssService ossService;
private final KnowledgeRetrievalService knowledgeRetrievalService;
@Override
public KnowledgeAttachVo queryById(Long id) {
@@ -126,18 +132,44 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
@Override
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
// 删除附件前，同步清理其片段记录与向量库中的向量
List<KnowledgeAttach> attaches = baseMapper.selectByIds(ids);
for (KnowledgeAttach attach : attaches) {
String docId = attach.getDocId();
String kid = String.valueOf(attach.getKnowledgeId());
vectorStoreService.removeByDocId(docId, kid);
knowledgeFragmentMapper.delete(
Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
if (attach.getOssId() != null) {
ossService.deleteFile(attach.getOssId());
}
knowledgeRetrievalService.invalidateKnowledge(kid);
}
return baseMapper.deleteByIds(ids) > 0;
}
@Override
public void upload(KnowledgeInfoUploadBo bo) {
MultipartFile file = bo.getFile();
final String fileHash;
try (InputStream input = file.getInputStream()) {
fileHash = DigestUtil.sha256Hex(input);
} catch (Exception e) {
throw new ServiceException("计算文件摘要失败", e);
}
boolean duplicate = baseMapper.exists(Wrappers.<KnowledgeAttach>lambdaQuery()
.eq(KnowledgeAttach::getKnowledgeId, bo.getKnowledgeId())
.eq(KnowledgeAttach::getFileHash, fileHash));
if (duplicate) {
throw new ServiceException("该文件已上传，请勿重复提交");
}
OssDTO ossDTO = ossService.uploadFile(file);
KnowledgeAttach knowledgeAttach = new KnowledgeAttach();
knowledgeAttach.setKnowledgeId(bo.getKnowledgeId());
knowledgeAttach.setOssId(ossDTO.getOssId());
knowledgeAttach.setDocId(RandomUtil.randomString(10));
knowledgeAttach.setFileHash(fileHash);
knowledgeAttach.setName(ossDTO.getOriginalName());
knowledgeAttach.setType(ossDTO.getFileSuffix());
knowledgeAttach.setStatus(KnowledgeAttachStatus.WAITING.getCode()); // 待解析
@@ -154,10 +186,17 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
@Override
public void parse(Long id) {
KnowledgeAttach attach = baseMapper.selectById(id);
if (attach == null || (!KnowledgeAttachStatus.WAITING.getCode().equals(attach.getStatus()) && !KnowledgeAttachStatus.FAILED.getCode().equals(attach.getStatus()))) {
if (attach == null || KnowledgeAttachStatus.PARSING.getCode().equals(attach.getStatus())) {
return;
}
int claimed = baseMapper.update(null, Wrappers.<KnowledgeAttach>lambdaUpdate()
.set(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode())
.set(KnowledgeAttach::getRemark, null)
.eq(KnowledgeAttach::getId, id)
.ne(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode()));
if (claimed == 0) return;
try {
attach.setStatus(KnowledgeAttachStatus.PARSING.getCode()); // 解析中
baseMapper.updateById(attach);
@@ -166,6 +205,18 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
Long knowledgeId = attach.getKnowledgeId();
String docId = attach.getDocId();
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
if (knowledgeInfoVo == null) {
throw new ServiceException("知识库不存在: " + knowledgeId);
}
int blockSize = knowledgeInfoVo.getTextBlockSize() == null
? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : knowledgeInfoVo.getTextBlockSize().intValue();
int overlap = knowledgeInfoVo.getOverlapChar() == null
? DocumentSplitConfig.DEFAULT_OVERLAP : knowledgeInfoVo.getOverlapChar().intValue();
DocumentSplitConfig splitConfig = new DocumentSplitConfig(
knowledgeInfoVo.getSeparator(), blockSize, overlap, attach.getType());
List<KnowledgeFragment> oldFragments = knowledgeFragmentMapper.selectList(
Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
// 获取文件信息并下载
List<OssDTO> ossDTOs = ossService.selectByIds(String.valueOf(attach.getOssId()));
@@ -178,28 +229,27 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
try (InputStream inputStream = new URL(ossDTO.getUrl()).openStream()) {
content = resourceLoader.getContent(inputStream);
}
List<String> chunkList = resourceLoader.getChunkList(content, String.valueOf(knowledgeId));
List<String> chunkList = resourceLoader.getChunkList(content, splitConfig);
List<String> fids = new ArrayList<>();
List<KnowledgeFragment> knowledgeFragmentList = new ArrayList<>();
if (CollUtil.isNotEmpty(chunkList)) {
for (int i = 0; i < chunkList.size(); i++) {
String fid = RandomUtil.randomString(10);
fids.add(fid);
KnowledgeFragment knowledgeFragment = new KnowledgeFragment();
knowledgeFragment.setKnowledgeId(knowledgeId);
knowledgeFragment.setDocId(docId);
knowledgeFragment.setIdx(i);
knowledgeFragment.setContent(chunkList.get(i));
knowledgeFragment.setCreateTime(new Date());
knowledgeFragmentList.add(knowledgeFragment);
}
knowledgeFragmentMapper.delete(Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
knowledgeFragmentMapper.insertBatch(knowledgeFragmentList);
log.info("文档切片并入库完成，共计 {} 个片段。id: {}", chunkList.size(), id);
if (CollUtil.isEmpty(chunkList)) {
throw new RuntimeException("文档分片结果为空，请检查文档内容或分片器是否支持该文件类型");
}
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
// 重新解析前先清理旧的向量数据，避免向量重复累积
List<String> fids = new ArrayList<>();
List<KnowledgeFragment> knowledgeFragmentList = new ArrayList<>();
for (int i = 0; i < chunkList.size(); i++) {
String fid = RandomUtil.randomString(10);
fids.add(fid);
KnowledgeFragment knowledgeFragment = new KnowledgeFragment();
knowledgeFragment.setKnowledgeId(knowledgeId);
knowledgeFragment.setDocId(docId);
knowledgeFragment.setFid(fid);
knowledgeFragment.setIdx(i);
knowledgeFragment.setContent(chunkList.get(i));
knowledgeFragment.setCreateTime(new Date());
knowledgeFragmentList.add(knowledgeFragment);
}
ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel());
StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo();
@@ -211,7 +261,27 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
storeEmbeddingBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModel());
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
try {
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
for (KnowledgeFragment old : oldFragments) {
if (StringUtils.isNotBlank(old.getFid())) {
vectorStoreService.removeByFid(old.getFid(), String.valueOf(knowledgeId));
}
}
} catch (Exception vectorError) {
for (String newFid : fids) {
try {
vectorStoreService.removeByFid(newFid, String.valueOf(knowledgeId));
} catch (Exception cleanupError) {
log.error("补偿删除新向量失败, kid={}, fid={}", knowledgeId, newFid, cleanupError);
}
}
throw vectorError;
}
knowledgeFragmentMapper.delete(Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
knowledgeFragmentMapper.insertBatch(knowledgeFragmentList);
knowledgeRetrievalService.invalidateKnowledge(String.valueOf(knowledgeId));
attach.setStatus(KnowledgeAttachStatus.COMPLETED.getCode()); // 已完成
baseMapper.updateById(attach);
@@ -223,4 +293,22 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
baseMapper.updateById(attach);
}
}
@Override
public KnowledgeReparseVo reparseKnowledge(Long knowledgeId) {
List<KnowledgeAttach> attachments = baseMapper.selectList(
Wrappers.<KnowledgeAttach>lambdaQuery().eq(KnowledgeAttach::getKnowledgeId, knowledgeId));
int submitted = 0;
int skipped = 0;
IKnowledgeAttachService proxy = SpringUtils.getBean(IKnowledgeAttachService.class);
for (KnowledgeAttach attachment : attachments) {
if (KnowledgeAttachStatus.PARSING.getCode().equals(attachment.getStatus())) {
skipped++;
} else {
proxy.parse(attachment.getId());
submitted++;
}
}
return new KnowledgeReparseVo(submitted, skipped, attachments.size());
}
}

15
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeFragmentServiceImpl.java
View File

@@ -40,6 +40,7 @@ public class KnowledgeFragmentServiceImpl implements IKnowledgeFragmentService {
private final IKnowledgeInfoService knowledgeInfoService;
private final IChatModelService chatModelService;
private final KnowledgeRetrievalService knowledgeRetrievalService;
private final org.ruoyi.service.vector.VectorStoreService vectorStoreService;
/**
* 查询知识片段
@@ -114,7 +115,11 @@ public class KnowledgeFragmentServiceImpl implements IKnowledgeFragmentService {
public Boolean updateByBo(KnowledgeFragmentBo bo) {
KnowledgeFragment update = MapstructUtils.convert(bo, KnowledgeFragment.class);
validEntityBeforeSave(update);
return baseMapper.updateById(update) > 0;
boolean updated = baseMapper.updateById(update) > 0;
if (updated && update.getKnowledgeId() != null) {
knowledgeRetrievalService.invalidateKnowledge(String.valueOf(update.getKnowledgeId()));
}
return updated;
}
/**
@@ -136,6 +141,14 @@ public class KnowledgeFragmentServiceImpl implements IKnowledgeFragmentService {
if(isValid){
//TODO 做一些业务上的校验,判断是否需要校验
}
// 删除 DB 片段前，同步删除向量库中对应向量
List<KnowledgeFragment> fragments = baseMapper.selectByIds(ids);
for (KnowledgeFragment fragment : fragments) {
if (StringUtils.isNotBlank(fragment.getFid()) && fragment.getKnowledgeId() != null) {
vectorStoreService.removeByFid(fragment.getFid(), String.valueOf(fragment.getKnowledgeId()));
knowledgeRetrievalService.invalidateKnowledge(String.valueOf(fragment.getKnowledgeId()));
}
}
return baseMapper.deleteByIds(ids) > 0;
}

52
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeInfoServiceImpl.java
View File

@@ -16,6 +16,10 @@ import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper;
import org.ruoyi.mapper.knowledge.KnowledgeInfoMapper;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.common.core.service.OssService;
import java.util.List;
import java.util.Map;
@@ -36,6 +40,12 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
private final KnowledgeAttachMapper knowledgeAttachMapper;
private final org.ruoyi.mapper.knowledge.KnowledgeFragmentMapper knowledgeFragmentMapper;
private final org.ruoyi.service.vector.VectorStoreService vectorStoreService;
private final KnowledgeRetrievalService knowledgeRetrievalService;
private final OssService ossService;
/**
* 查询知识库
*
@@ -97,10 +107,14 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
*/
private void fillDocumentCount(List<KnowledgeInfoVo> records) {
if (records == null || records.isEmpty()) return;
for (KnowledgeInfoVo vo : records) {
int count = knowledgeAttachMapper.countByKnowledgeId(vo.getId());
vo.setDocumentCount(count);
List<Long> ids = records.stream().map(KnowledgeInfoVo::getId).toList();
Map<Long, Integer> counts = new java.util.HashMap<>();
for (Map<String, Object> row : knowledgeAttachMapper.countByKnowledgeIds(ids)) {
Number kid = (Number) (row.get("knowledgeId") != null ? row.get("knowledgeId") : row.get("knowledgeid"));
Number count = (Number) (row.get("documentCount") != null ? row.get("documentCount") : row.get("documentcount"));
if (kid != null && count != null) counts.put(kid.longValue(), count.intValue());
}
records.forEach(vo -> vo.setDocumentCount(counts.getOrDefault(vo.getId(), 0)));
}
/**
@@ -130,13 +144,20 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
public Boolean updateByBo(KnowledgeInfoBo bo) {
KnowledgeInfo update = MapstructUtils.convert(bo, KnowledgeInfo.class);
validEntityBeforeSave(update);
return baseMapper.updateById(update) > 0;
boolean updated = baseMapper.updateById(update) > 0;
if (updated) knowledgeRetrievalService.invalidateKnowledge(String.valueOf(bo.getId()));
return updated;
}
/**
* 保存前的数据校验
*/
private void validEntityBeforeSave(KnowledgeInfo entity){
int blockSize = entity.getTextBlockSize() == null
? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : entity.getTextBlockSize().intValue();
int overlap = entity.getOverlapChar() == null
? DocumentSplitConfig.DEFAULT_OVERLAP : entity.getOverlapChar().intValue();
new DocumentSplitConfig(entity.getSeparator(), blockSize, overlap, "");
//TODO 做一些数据校验,如唯一约束
}
@@ -148,10 +169,33 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
* @return 是否删除成功
*/
@Override
@Transactional(rollbackFor = Exception.class)
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
if(isValid){
//TODO 做一些业务上的校验,判断是否需要校验
}
for (Long kid : ids) {
KnowledgeInfo info = baseMapper.selectById(kid);
// 1. 删除向量库中该知识库的所有向量（按文档逐个清理，三种向量库行为一致）
List<org.ruoyi.domain.entity.knowledge.KnowledgeAttach> attaches = knowledgeAttachMapper.selectList(
Wrappers.lambdaQuery(org.ruoyi.domain.entity.knowledge.KnowledgeAttach.class)
.eq(org.ruoyi.domain.entity.knowledge.KnowledgeAttach::getKnowledgeId, kid));
vectorStoreService.removeById(String.valueOf(kid), info == null ? null : info.getVectorModel());
List<Long> ossIds = attaches.stream()
.map(org.ruoyi.domain.entity.knowledge.KnowledgeAttach::getOssId)
.filter(java.util.Objects::nonNull).toList();
if (!ossIds.isEmpty()) {
for (Long ossId : ossIds) {
ossService.deleteFile(ossId);
}
}
// 2. 删除该知识库下的附件与片段记录
knowledgeAttachMapper.delete(Wrappers.lambdaQuery(org.ruoyi.domain.entity.knowledge.KnowledgeAttach.class)
.eq(org.ruoyi.domain.entity.knowledge.KnowledgeAttach::getKnowledgeId, kid));
knowledgeFragmentMapper.delete(Wrappers.lambdaQuery(org.ruoyi.domain.entity.knowledge.KnowledgeFragment.class)
.eq(org.ruoyi.domain.entity.knowledge.KnowledgeFragment::getKnowledgeId, kid));
knowledgeRetrievalService.invalidateKnowledge(String.valueOf(kid));
}
return baseMapper.deleteByIds(ids) > 0;
}
}

11
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CodeFileLoader.java
View File

@@ -3,13 +3,16 @@ package org.ruoyi.service.knowledge.impl.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.ruoyi.common.core.exception.ServiceException;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.List;
@Component
@@ -21,20 +24,20 @@ public class CodeFileLoader implements ResourceLoader {
@Override
public String getContent(InputStream inputStream) {
StringBuffer stringBuffer = new StringBuffer();
try (InputStreamReader reader = new InputStreamReader(inputStream);
try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
BufferedReader bufferedReader = new BufferedReader(reader)) {
String line;
while ((line = bufferedReader.readLine()) != null) {
stringBuffer.append(line).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
throw new ServiceException("读取代码文件失败", e);
}
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CsvFileLoader.java
View File

@@ -1,6 +1,7 @@
package org.ruoyi.service.knowledge.impl.loader;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.io.InputStream;
import java.util.List;
@@ -12,7 +13,7 @@ public class CsvFileLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return null;
}
}

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/ExcelFileLoader.java
View File

@@ -6,6 +6,7 @@ import dev.langchain4j.data.document.parser.apache.tika.ApacheTikaDocumentParser
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
@@ -38,7 +39,7 @@ public class ExcelFileLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/FolderLoader.java
View File

@@ -1,6 +1,7 @@
package org.ruoyi.service.knowledge.impl.loader;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.io.InputStream;
import java.util.List;
@@ -12,7 +13,7 @@ public class FolderLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return null;
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/GithubLoader.java
View File

@@ -1,6 +1,7 @@
package org.ruoyi.service.knowledge.impl.loader;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.io.InputStream;
import java.util.List;
@@ -12,7 +13,7 @@ public class GithubLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return null;
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/JsonFileLoader.java
View File

@@ -1,6 +1,7 @@
package org.ruoyi.service.knowledge.impl.loader;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.io.InputStream;
import java.util.List;
@@ -12,7 +13,7 @@ public class JsonFileLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return null;
}
}

11
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/MarkDownFileLoader.java
View File

@@ -3,13 +3,16 @@ package org.ruoyi.service.knowledge.impl.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.ruoyi.common.core.exception.ServiceException;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.List;
@Component
@@ -21,20 +24,20 @@ public class MarkDownFileLoader implements ResourceLoader {
@Override
public String getContent(InputStream inputStream) {
StringBuffer stringBuffer = new StringBuffer();
try (InputStreamReader reader = new InputStreamReader(inputStream);
try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
BufferedReader bufferedReader = new BufferedReader(reader)) {
String line;
while ((line = bufferedReader.readLine()) != null) {
stringBuffer.append(line).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
throw new ServiceException("读取 Markdown 文件失败", e);
}
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/PdfFileLoader.java
View File

@@ -6,6 +6,7 @@ import org.apache.pdfbox.io.RandomAccessReadBuffer;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
@@ -32,7 +33,7 @@ public class PdfFileLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
return characterTextSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return characterTextSplitter.split(content, config);
}
}

12
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/TextFileLoader.java
View File

@@ -3,7 +3,9 @@ package org.ruoyi.service.knowledge.impl.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.ruoyi.common.core.exception.ServiceException;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
@@ -22,18 +24,16 @@ public class TextFileLoader implements ResourceLoader {
@Override
public String getContent(InputStream inputStream) {
String stringBuffer = "";
try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
BufferedReader bufferedReader = new BufferedReader(reader)) {
stringBuffer = bufferedReader.lines().collect(Collectors.joining());
return bufferedReader.lines().collect(Collectors.joining("\n"));
} catch (IOException e) {
e.printStackTrace();
throw new ServiceException("读取文本文件失败", e);
}
return stringBuffer;
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/WordLoader.java
View File

@@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
@@ -32,8 +33,8 @@ public class WordLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

71
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java
View File

@@ -1,82 +1,17 @@
package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Component
@Slf4j
@Primary
@AllArgsConstructor
public class CharacterTextSplitter implements TextSplitter {
private final IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content, String kid) {
// 默认配置值
String knowledgeSeparator = "#";
int textBlockSize = 1000;
int overlapChar = 50;
// 根据知识库ID查询配置，覆盖默认值
if (StringUtils.isNotBlank(kid)) {
try {
KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
if (info != null) {
if (StringUtils.isNotBlank(info.getSeparator())) {
knowledgeSeparator = info.getSeparator();
}
if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
textBlockSize = info.getTextBlockSize().intValue();
}
if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
overlapChar = info.getOverlapChar().intValue();
}
}
} catch (Exception e) {
log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
}
}
List<String> chunkList = new ArrayList<>();
if (content.contains(knowledgeSeparator) && StringUtils.isNotBlank(knowledgeSeparator)) {
// 按自定义分隔符切分
String[] chunks = content.split(knowledgeSeparator);
chunkList.addAll(Arrays.asList(chunks));
} else {
int indexMin = 0;
int len = content.length();
int i = 0;
int right = 0;
while (true) {
if (len > right) {
int begin = i * textBlockSize - overlapChar;
if (begin < indexMin) {
begin = indexMin;
}
int end = textBlockSize * (i + 1) + overlapChar;
if (end > len) {
end = len;
}
String chunk = content.substring(begin, end);
chunkList.add(chunk);
i++;
right = right + textBlockSize;
} else {
break;
}
}
}
return chunkList;
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, SplitterSupport::paragraphs);
}
}

9
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java
View File

@@ -1,18 +1,15 @@
package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class CodeTextSplitter implements TextSplitter {
@Override
public List<String> split(String content, String kid) {
return null;
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, SplitterSupport::paragraphs);
}
}

70
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/ExcelTextSplitter.java
View File

@@ -1,79 +1,15 @@
package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class ExcelTextSplitter implements TextSplitter {
private final IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content, String kid) {
// 默认配置
String knowledgeSeparator = "#";
int textBlockSize = 1000;
int overlapChar = 50;
// 根据知识库ID查询配置，覆盖默认值
if (StringUtils.isNotBlank(kid)) {
try {
KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
if (info != null) {
if (StringUtils.isNotBlank(info.getSeparator())) {
knowledgeSeparator = info.getSeparator();
}
if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
textBlockSize = info.getTextBlockSize().intValue();
}
if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
overlapChar = info.getOverlapChar().intValue();
}
}
} catch (Exception e) {
log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
}
}
List<String> chunkList = new ArrayList<>();
if (content.contains(knowledgeSeparator) && StringUtils.isNotBlank(knowledgeSeparator)) {
// 按自定义分隔符切分
String[] chunks = content.split(knowledgeSeparator);
chunkList.addAll(Arrays.asList(chunks));
} else {
int indexMin = 0;
int len = content.length();
int i = 0;
int right = 0;
while (true) {
if (len > right) {
int begin = i * textBlockSize - overlapChar;
if (begin < indexMin) {
begin = indexMin;
}
int end = textBlockSize * (i + 1) + overlapChar;
if (end > len) {
end = len;
}
String chunk = content.substring(begin, end);
chunkList.add(chunk);
i++;
right = right + textBlockSize;
} else {
break;
}
}
}
return chunkList;
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, SplitterSupport::lines);
}
}

50
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java
View File

@@ -1,18 +1,56 @@
package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class MarkdownTextSplitter implements TextSplitter {
@Override
public List<String> split(String content, String kid) {
return null;
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, this::sections);
}
/** Split headings without treating heading-looking lines inside fenced code as headings. */
private List<String> sections(String markdown) {
String[] lines = markdown.split("\\n", -1);
List<String> sections = new ArrayList<>();
StringBuilder current = new StringBuilder();
boolean fenced = false;
String fenceMarker = null;
for (int i = 0; i < lines.length; i++) {
String line = lines[i];
String trimmed = line.stripLeading();
if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) {
String marker = trimmed.substring(0, 3);
if (!fenced) {
fenced = true;
fenceMarker = marker;
} else if (marker.equals(fenceMarker)) {
fenced = false;
fenceMarker = null;
}
}
boolean atx = !fenced && line.matches("^#{1,6}(\\s+.*)?$");
boolean setextTitle = !fenced && i + 1 < lines.length
&& lines[i + 1].matches("^\\s*(=+|-+)\\s*$") && !line.isBlank();
if ((atx || setextTitle) && current.length() > 0) flush(sections, current);
current.append(line);
if (i < lines.length - 1) current.append('\n');
if (setextTitle) {
current.append(lines[++i]);
if (i < lines.length - 1) current.append('\n');
}
}
flush(sections, current);
return sections;
}
private static void flush(List<String> sections, StringBuilder current) {
if (!current.toString().isBlank()) sections.add(current.toString());
current.setLength(0);
}
}

67
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java Normal file
View File

@@ -0,0 +1,67 @@
package org.ruoyi.service.knowledge.impl.split;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import java.util.regex.Pattern;
/** Shared, deterministic split pipeline used by every supported document format. */
public final class SplitterSupport {
private SplitterSupport() {}
public static List<String> split(String content, DocumentSplitConfig config,
Function<String, List<String>> naturalSections) {
if (content == null || content.isBlank()) return List.of();
String normalized = content.replace("\r\n", "\n").replace('\r', '\n');
List<String> primary = literalSections(normalized, config.separator());
List<String> result = new ArrayList<>();
for (String part : primary) {
List<String> sections = naturalSections.apply(part);
String joined = sections.stream().filter(s -> s != null && !s.isBlank())
.map(String::strip).reduce((a, b) -> a + "\n\n" + b).orElse("");
result.addAll(slidingWindow(joined, config.blockSize(), config.overlap()));
}
return result;
}
static List<String> literalSections(String content, String separator) {
if (separator == null || separator.isEmpty() || !content.contains(separator)) return List.of(content);
List<String> parts = new ArrayList<>();
for (String part : content.split(Pattern.quote(separator), -1)) {
if (!part.isBlank()) parts.add(part);
}
return parts;
}
/** blockSize is a strict maximum; overlap is the repeated suffix/prefix length. */
public static List<String> slidingWindow(String content, int blockSize, int overlap) {
if (content == null || content.isBlank()) return List.of();
String value = content.strip();
List<String> chunks = new ArrayList<>();
int step = blockSize - overlap;
for (int start = 0; start < value.length(); start += step) {
int end = Math.min(value.length(), start + blockSize);
String chunk = value.substring(start, end).strip();
if (!chunk.isEmpty()) chunks.add(chunk);
if (end == value.length()) break;
}
return chunks;
}
/** Compatibility helper retained for callers/tests; now enforces strict maximum size. */
public static List<String> mergeAndSplit(String[] sections, int blockSize, int overlap) {
return split(String.join("\n\n", sections),
new DocumentSplitConfig(null, blockSize, overlap, ""),
text -> List.of(text));
}
static List<String> paragraphs(String content) {
return List.of(content.split("\\n\\s*\\n+"));
}
static List<String> lines(String content) {
return List.of(content.split("\\n+"));
}
}

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/TokenTextSplitter.java
View File

@@ -3,6 +3,7 @@ package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.TextSplitter;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.springframework.stereotype.Component;
import java.util.List;
@@ -12,7 +13,7 @@ import java.util.List;
@Slf4j
public class TokenTextSplitter implements TextSplitter {
@Override
public List<String> split(String content, String kid) {
return null;
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, SplitterSupport::paragraphs);
}
}

11
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/retriever/CustomVectorRetriever.java
View File

@@ -1,6 +1,7 @@
package org.ruoyi.service.knowledge.retriever;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.data.document.Metadata;
import dev.langchain4j.rag.content.Content;
import dev.langchain4j.rag.content.retriever.ContentRetriever;
import dev.langchain4j.rag.query.Query;
@@ -55,11 +56,17 @@ public class CustomVectorRetriever implements ContentRetriever {
queryVectorBo.setRerankScoreThreshold(knowledgeInfoVo.getRerankScoreThreshold());
// 通过统一服务执行检索
List<String> nearestList = knowledgeRetrievalService.retrieveTexts(queryVectorBo);
var nearestList = knowledgeRetrievalService.retrieve(queryVectorBo);
// 将结果包装为标准的 Content 返回
return nearestList.stream()
.map(text -> Content.from(TextSegment.from(text)))
.map(vo -> {
Metadata metadata = new Metadata();
metadata.put("kid", String.valueOf(knowledgeInfoVo.getId()));
metadata.put("docId", Objects.toString(vo.getDocId(), ""));
metadata.put("fid", Objects.toString(vo.getId(), ""));
return Content.from(TextSegment.from(vo.getContent(), metadata));
})
.collect(Collectors.toList());
}
}

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/rerank/impl/AliBaiLianRerankModelService.java
View File

@@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit;
*/
@Slf4j
@Component("qianwenRerank")
@org.springframework.context.annotation.Scope("prototype")
public class AliBaiLianRerankModelService implements RerankModelService {
private final OkHttpClient okHttpClient;

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/rerank/impl/SiliconFlowRerankModelService.java
View File

@@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;
*/
@Slf4j
@Component("siliconflowRerank")
@org.springframework.context.annotation.Scope("prototype")
public class SiliconFlowRerankModelService implements RerankModelService {
private static final String DEFAULT_BASE_URL = "https://api.siliconflow.cn/v1/rerank";

1
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/rerank/impl/ZhiPuRerankModelService.java
View File

@@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit;
*/
@Slf4j
@Component("zhipuRerank")
@org.springframework.context.annotation.Scope("prototype")
public class ZhiPuRerankModelService implements RerankModelService {
private final OkHttpClient okHttpClient;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/retrieval/KnowledgeRetrievalService.java
View File

@@ -31,4 +31,6 @@ public interface KnowledgeRetrievalService {
* @return 检索结果列表
*/
List<KnowledgeRetrievalVo> retrieve(QueryVectorBo queryVectorBo);
void invalidateKnowledge(String kid);
}

83
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/retrieval/impl/KnowledgeRetrievalServiceImpl.java
View File

@@ -3,6 +3,7 @@ package org.ruoyi.service.retrieval.impl;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.common.core.exception.ServiceException;
import org.ruoyi.domain.bo.rerank.RerankRequest;
import org.ruoyi.domain.bo.rerank.RerankResult;
import org.ruoyi.domain.bo.vector.QueryVectorBo;
@@ -17,6 +18,8 @@ import org.springframework.stereotype.Service;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
@@ -40,6 +43,9 @@ public class KnowledgeRetrievalServiceImpl implements KnowledgeRetrievalService
* 如果启用重排序，粗召回会获取更多结果供重排序筛选
*/
private static final int RERANK_EXPANSION_FACTOR = 3;
private static final long CACHE_TTL_MILLIS = TimeUnit.MINUTES.toMillis(5);
private static final int CACHE_MAX_ENTRIES = 1000;
private final Map<String, CacheEntry> retrievalCache = new ConcurrentHashMap<>();
@Override
public List<String> retrieveTexts(QueryVectorBo queryVectorBo) {
@@ -51,6 +57,11 @@ public class KnowledgeRetrievalServiceImpl implements KnowledgeRetrievalService
@Override
public List<KnowledgeRetrievalVo> retrieve(QueryVectorBo queryVectorBo) {
String cacheKey = cacheKey(queryVectorBo);
CacheEntry cached = retrievalCache.get(cacheKey);
if (cached != null && System.currentTimeMillis() - cached.createdAt < CACHE_TTL_MILLIS) {
return copyResults(cached.results);
}
log.info("开始知识库检索, kid={}, query={}", queryVectorBo.getKid(), queryVectorBo.getQuery());
// 1. 粗召回阶段 (向量检索 + 关键词搜索)
@@ -68,18 +79,21 @@ public class KnowledgeRetrievalServiceImpl implements KnowledgeRetrievalService
// 3. 重排序阶段 (可选)
List<KnowledgeRetrievalVo> finalResults = coarseResults;
if (Boolean.TRUE.equals(queryVectorBo.getEnableRerank()) &&
StringUtils.isNotBlank(queryVectorBo.getRerankModelName())) {
boolean rerankApplied = Boolean.TRUE.equals(queryVectorBo.getEnableRerank()) &&
StringUtils.isNotBlank(queryVectorBo.getRerankModelName());
if (rerankApplied) {
finalResults = performRerank(queryVectorBo, coarseResults);
}
// 4. 应用分值阈值过滤 (重排分值或 RRF 分值)
double threshold = queryVectorBo.getRerankScoreThreshold() != null ?
queryVectorBo.getRerankScoreThreshold() : 0.0;
return finalResults.stream()
.filter(res -> res.getScore() >= threshold)
.collect(Collectors.toList());
if (rerankApplied && queryVectorBo.getRerankScoreThreshold() != null) {
double threshold = queryVectorBo.getRerankScoreThreshold();
finalResults = finalResults.stream()
.filter(res -> res.getScore() != null && res.getScore() >= threshold)
.collect(Collectors.toList());
}
cache(cacheKey, finalResults);
return copyResults(finalResults);
}
/**
@@ -132,7 +146,8 @@ public class KnowledgeRetrievalServiceImpl implements KnowledgeRetrievalService
List<KnowledgeFragmentVo> fragments = fragmentMapper.searchByKeyword(kid, queryVectorBo.getQuery(), finalTargetMaxResults);
return fragments.stream().map(f -> {
KnowledgeRetrievalVo vo = new KnowledgeRetrievalVo();
vo.setId(f.getId().toString());
// 优先使用 fid 作为融合标识（与向量侧一致），历史数据无 fid 时回退主键
vo.setId(StringUtils.isNotBlank(f.getFid()) ? f.getFid() : f.getId().toString());
vo.setContent(f.getContent());
vo.setDocId(f.getDocId());
vo.setIdx(f.getIdx());
@@ -155,7 +170,11 @@ public class KnowledgeRetrievalServiceImpl implements KnowledgeRetrievalService
} catch (Exception e) {
log.error("混合检索执行失败，回退到纯向量检索: {}", e.getMessage(), e);
return vectorStoreService.search(copyOf(queryVectorBo, targetMaxResults));
try {
return vectorStoreService.search(copyOf(queryVectorBo, targetMaxResults));
} catch (Exception vectorError) {
throw new ServiceException("知识库检索不可用：向量与混合检索均失败");
}
}
}
@@ -182,19 +201,21 @@ public class KnowledgeRetrievalServiceImpl implements KnowledgeRetrievalService
RerankResult rerankResult = rerankModel.rerank(rerankRequest);
// 写回分数并记录原始分
List<KnowledgeRetrievalVo> reranked = new ArrayList<>();
for (RerankResult.RerankDocument doc : rerankResult.getDocuments()) {
if (doc.getIndex() != null && doc.getIndex() < coarseResults.size()) {
KnowledgeRetrievalVo vo = coarseResults.get(doc.getIndex());
vo.setRawScore(vo.getScore());
vo.setScore(doc.getRelevanceScore());
reranked.add(vo);
}
}
// 按新分排序
coarseResults.sort((a, b) -> b.getScore().compareTo(a.getScore()));
reranked.sort((a, b) -> b.getScore().compareTo(a.getScore()));
// 截断到 topN
return coarseResults.subList(0, Math.min(topN, coarseResults.size()));
return reranked.subList(0, Math.min(topN, reranked.size()));
} catch (Exception e) {
log.error("重排序流程失败: {}", e.getMessage());
@@ -253,4 +274,42 @@ public class KnowledgeRetrievalServiceImpl implements KnowledgeRetrievalService
copy.setBaseUrl(original.getBaseUrl());
return copy;
}
@Override
public void invalidateKnowledge(String kid) {
if (StringUtils.isBlank(kid)) {
retrievalCache.clear();
} else {
retrievalCache.keySet().removeIf(key -> key.startsWith(kid + "|"));
}
}
private void cache(String key, List<KnowledgeRetrievalVo> results) {
if (retrievalCache.size() >= CACHE_MAX_ENTRIES) {
long now = System.currentTimeMillis();
retrievalCache.entrySet().removeIf(e -> now - e.getValue().createdAt >= CACHE_TTL_MILLIS);
if (retrievalCache.size() >= CACHE_MAX_ENTRIES) {
retrievalCache.clear();
}
}
retrievalCache.put(key, new CacheEntry(System.currentTimeMillis(), copyResults(results)));
}
private String cacheKey(QueryVectorBo bo) {
return String.join("|", Objects.toString(bo.getKid(), ""), Objects.toString(bo.getQuery(), ""),
Objects.toString(bo.getMaxResults(), ""), Objects.toString(bo.getVectorModelName(), ""),
Objects.toString(bo.getEmbeddingModelName(), ""), Objects.toString(bo.getSimilarityThreshold(), ""),
Objects.toString(bo.getEnableHybrid(), ""), Objects.toString(bo.getHybridAlpha(), ""),
Objects.toString(bo.getEnableRerank(), ""), Objects.toString(bo.getRerankModelName(), ""),
Objects.toString(bo.getRerankTopN(), ""), Objects.toString(bo.getRerankScoreThreshold(), ""));
}
private List<KnowledgeRetrievalVo> copyResults(List<KnowledgeRetrievalVo> source) {
return source.stream().map(vo -> KnowledgeRetrievalVo.builder()
.id(vo.getId()).docId(vo.getDocId()).knowledgeId(vo.getKnowledgeId()).idx(vo.getIdx())
.content(vo.getContent()).score(vo.getScore()).originalIndex(vo.getOriginalIndex())
.rawScore(vo.getRawScore()).sourceName(vo.getSourceName()).build()).collect(Collectors.toList());
}
private record CacheEntry(long createdAt, List<KnowledgeRetrievalVo> results) { }
}

67
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/vector/impl/MilvusVectorStoreStrategy.java
View File

@@ -12,6 +12,9 @@ import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder;
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
import io.milvus.param.IndexType;
import io.milvus.param.MetricType;
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.service.collection.request.DropCollectionReq;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
@@ -37,13 +40,16 @@ import java.util.stream.IntStream;
public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
private final KnowledgeAttachMapper knowledgeAttachMapper;
private final org.ruoyi.mapper.knowledge.KnowledgeInfoMapper knowledgeInfoMapper;
public MilvusVectorStoreStrategy(VectorStoreProperties vectorStoreProperties,
IChatModelService chatModelService,
EmbeddingModelFactory embeddingModelFactory,
KnowledgeAttachMapper knowledgeAttachMapper) {
KnowledgeAttachMapper knowledgeAttachMapper,
org.ruoyi.mapper.knowledge.KnowledgeInfoMapper knowledgeInfoMapper) {
super(vectorStoreProperties, embeddingModelFactory, chatModelService);
this.knowledgeAttachMapper = knowledgeAttachMapper;
this.knowledgeInfoMapper = knowledgeInfoMapper;
}
// 缓存不同集合与 autoFlush 配置的 Milvus 连接
@@ -100,9 +106,11 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
long startTime = System.currentTimeMillis();
// 复用连接，写入场景使用 autoFlush=false 以提升批量插入性能
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, dimension, false);
// addAll is already batched; flush before returning so newly parsed fragments are immediately searchable.
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, dimension, true);
IntStream.range(0, chunkList.size()).forEach(i -> {
List<TextSegment> segments = new ArrayList<>(chunkList.size());
for (int i = 0; i < chunkList.size(); i++) {
String text = chunkList.get(i);
String fid = fidList.get(i);
Metadata metadata = new Metadata();
@@ -110,13 +118,18 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
metadata.put("kid", kid);
metadata.put("docId", docId);
TextSegment textSegment = TextSegment.from(text, metadata);
Embedding embedding = embeddingModel.embed(text).content();
segments.add(TextSegment.from(text, metadata));
}
List<Embedding> embeddings = embeddingModel.embedAll(segments).content();
if (embeddings.size() != segments.size()) {
throw new org.ruoyi.common.core.exception.ServiceException("Embedding 返回数量与分片数量不一致");
}
for (Embedding embedding : embeddings) {
// 单位化处理
float[] vector = embedding.vector();
normalize(vector);
embeddingStore.add(Embedding.from(vector), textSegment);
});
}
embeddingStore.addAll(embeddings, segments);
long endTime = System.currentTimeMillis();
log.info("Milvus向量存储完成消耗时间：{}秒", (endTime - startTime) / 1000);
}
@@ -174,6 +187,7 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
if (segment == null) continue;
String docId = segment.metadata().getString("docId");
String fid = segment.metadata().getString("fid");
String sourceName = "未知来源";
if (docId != null) {
KnowledgeAttach attach = knowledgeAttachMapper.selectOne(new LambdaQueryWrapper<KnowledgeAttach>()
@@ -188,6 +202,8 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
double score = match.score();
resultList.add(org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo.builder()
.id(fid)
.docId(docId)
.content(segment.text())
.score(score)
.sourceName(sourceName)
@@ -200,16 +216,40 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
@SneakyThrows
public void removeById(String id, String modelName) {
// 注意：此处原逻辑使用 collectionname + id，保持现状
int dimension = getModelDimension(modelName);
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(vectorStoreProperties.getMilvus().getCollectionname() + id, dimension, false);
embeddingStore.remove(id);
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + id;
MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()
.uri(vectorStoreProperties.getMilvus().getUrl()).build());
try {
client.dropCollection(DropCollectionReq.builder().collectionName(collectionName).build());
storeCache.keySet().removeIf(key -> key.startsWith(collectionName));
log.info("Milvus collection deleted: {}", collectionName);
} catch (Exception e) {
log.error("Milvus collection delete failed: {}", collectionName, e);
throw new org.ruoyi.common.core.exception.ServiceException("Milvus向量集合删除失败");
} finally {
client.close();
}
}
/**
* 根据知识库ID解析其 embedding 模型维度，失败时回退默认 1024
*/
private int getDimensionByKid(String kid) {
try {
org.ruoyi.domain.entity.knowledge.KnowledgeInfo info = knowledgeInfoMapper.selectById(Long.parseLong(kid));
if (info != null && info.getEmbeddingModel() != null) {
return getModelDimension(info.getEmbeddingModel());
}
} catch (Exception e) {
log.warn("根据 kid={} 解析向量维度失败，使用默认 1024: {}", kid, e.getMessage());
}
return 1024;
}
@Override
public void removeByDocId(String docId, String kid) {
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
// 使用默认维度，因为删除操作不需要精确的维度信息
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, 1024, false);
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, getDimensionByKid(kid), true);
Filter filter = MetadataFilterBuilder.metadataKey("docId").isEqualTo(docId);
embeddingStore.removeAll(filter);
log.info("Milvus成功删除 docId={} 的所有向量数据", docId);
@@ -218,8 +258,7 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
@Override
public void removeByFid(String fid, String kid) {
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
// 使用默认维度，因为删除操作不需要精确的维度信息
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, 1024, false);
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, getDimensionByKid(kid), true);
Filter filter = MetadataFilterBuilder.metadataKey("fid").isEqualTo(fid);
embeddingStore.removeAll(filter);
log.info("Milvus成功删除 fid={} 的所有向量数据", fid);

Some files were not shown because too many files have changed in this diff Show More