From 9a2f326e42c07614f0898a1ff279176069a768f1 Mon Sep 17 00:00:00 2001 From: ageerle Date: Tue, 21 Jul 2026 14:33:28 +0800 Subject: [PATCH] =?UTF-8?q?fix(rag):=20=E5=90=88=E5=B9=B6=E5=90=8E?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3-=E5=8D=87=E7=BA=A7=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=20MySQL=208=E3=80=81=E9=87=8D=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E6=8C=89=20docId=20=E6=B8=85=E7=90=86=E6=97=A7?= =?UTF-8?q?=E5=90=91=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 升级脚本 2026-07-20-knowledge-fragment-fid.sql 使用了 MySQL 8 不支持的 ADD COLUMN IF NOT EXISTS（MariaDB 语法），在存量库执行会直接语法报错。 改为与脚本内索引守卫一致的 information_schema 判断 + PREPARE 方式，保持幂等。 2. parse() 重新解析时按 fragment.fid 删除旧向量：存量数据的 fid 为迁移脚本 回填的 MD5 值，与向量库中实际存储的随机 fid 不一致，首次重解析无法命中 旧向量，导致新旧向量重复累积。改为写入新向量前先按 docId 清理（三种 向量库策略均支持），写入失败时仍补偿删除新 fid。 Co-Authored-By: Claude --- .../2026-07-20-knowledge-fragment-fid.sql | 18 ++++++++++++++++-- .../impl/KnowledgeAttachServiceImpl.java | 11 ++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/script/sql/update/2026-07-20-knowledge-fragment-fid.sql b/docs/script/sql/update/2026-07-20-knowledge-fragment-fid.sql index 58b83adb..0bb5c264 100644 --- a/docs/script/sql/update/2026-07-20-knowledge-fragment-fid.sql +++ b/docs/script/sql/update/2026-07-20-knowledge-fragment-fid.sql @@ -1,6 +1,14 @@ -- RAG metadata migration (MySQL 8). Safe to execute repeatedly. +-- 注意：MySQL 8 不支持 ALTER TABLE ... ADD COLUMN IF NOT EXISTS（仅 MariaDB 支持）， +-- 因此列的增量添加统一用 information_schema 守卫 + PREPARE 实现幂等。 +SET @add_file_hash_col = IF(EXISTS( + SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE() + AND table_name = 'knowledge_attach' AND column_name = 'file_hash'), + 'SELECT 1', + 'ALTER TABLE `knowledge_attach` ADD COLUMN `file_hash` varchar(64) NULL DEFAULT NULL COMMENT ''文件SHA-256摘要'' AFTER `doc_id`'); +PREPARE stmt FROM @add_file_hash_col; EXECUTE stmt; DEALLOCATE PREPARE stmt; + ALTER TABLE `knowledge_attach` - ADD COLUMN IF NOT EXISTS `file_hash` varchar(64) NULL DEFAULT NULL COMMENT '文件SHA-256摘要' AFTER `doc_id`, MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID'; SET @add_file_hash = IF(EXISTS( @@ -9,8 +17,14 @@ SET @add_file_hash = IF(EXISTS( 'SELECT 1', 'ALTER TABLE `knowledge_attach` ADD UNIQUE INDEX `uk_knowledge_file_hash` (`knowledge_id`, `file_hash`)'); PREPARE stmt FROM @add_file_hash; EXECUTE stmt; DEALLOCATE PREPARE stmt; +SET @add_fid_col = IF(EXISTS( + SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE() + AND table_name = 'knowledge_fragment' AND column_name = 'fid'), + 'SELECT 1', + 'ALTER TABLE `knowledge_fragment` ADD COLUMN `fid` varchar(32) NULL DEFAULT NULL COMMENT ''向量库片段ID'' AFTER `id`'); +PREPARE stmt FROM @add_fid_col; EXECUTE stmt; DEALLOCATE PREPARE stmt; + ALTER TABLE `knowledge_fragment` - ADD COLUMN IF NOT EXISTS `fid` varchar(32) NULL DEFAULT NULL COMMENT '向量库片段ID' AFTER `id`, MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID'; UPDATE `knowledge_fragment` diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java index ed86abab..c36db0a2 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java @@ -215,8 +215,6 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService { ? DocumentSplitConfig.DEFAULT_OVERLAP : knowledgeInfoVo.getOverlapChar().intValue(); DocumentSplitConfig splitConfig = new DocumentSplitConfig( knowledgeInfoVo.getSeparator(), blockSize, overlap, attach.getType()); - List oldFragments = knowledgeFragmentMapper.selectList( - Wrappers.lambdaQuery().eq(KnowledgeFragment::getDocId, docId)); // 获取文件信息并下载 List ossDTOs = ossService.selectByIds(String.valueOf(attach.getOssId())); @@ -262,12 +260,11 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService { storeEmbeddingBo.setApiKey(chatModelVo.getApiKey()); storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost()); try { + // 写入新向量前，先按 docId 清理该文档的旧向量： + // 历史数据的片段 fid 为迁移脚本回填的 MD5 值，与向量库中实际存储的 fid 不一致， + // 按 fid 删除无法命中旧向量，会导致重复向量累积；按 docId 清理对三种向量库均一致有效。 + vectorStoreService.removeByDocId(docId, String.valueOf(knowledgeId)); vectorStoreService.storeEmbeddings(storeEmbeddingBo); - for (KnowledgeFragment old : oldFragments) { - if (StringUtils.isNotBlank(old.getFid())) { - vectorStoreService.removeByFid(old.getFid(), String.valueOf(knowledgeId)); - } - } } catch (Exception vectorError) { for (String newFid : fids) { try {