xieyinlu/ruoyi-ai
1
0
Fork 0
mirror of https://gitcode.com/ageerle/ruoyi-ai.git synced 2026-09-13 00:14:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(rag): 合并后修正-升级脚本兼容 MySQL 8、重解析按 docId 清理旧向量

Browse Source 
1. 升级脚本 2026-07-20-knowledge-fragment-fid.sql 使用了 MySQL 8 不支持的
   ADD COLUMN IF NOT EXISTS（MariaDB 语法），在存量库执行会直接语法报错。
   改为与脚本内索引守卫一致的 information_schema 判断 + PREPARE 方式，保持幂等。

2. parse() 重新解析时按 fragment.fid 删除旧向量：存量数据的 fid 为迁移脚本
   回填的 MD5 值，与向量库中实际存储的随机 fid 不一致，首次重解析无法命中
   旧向量，导致新旧向量重复累积。改为写入新向量前先按 docId 清理（三种
   向量库策略均支持），写入失败时仍补偿删除新 fid。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
ageerle
2026-07-21 14:33:28 +08:00
parent 46a8d6b552
commit 9a2f326e42
2 changed files with 20 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
docs/script/sql/update/2026-07-20-knowledge-fragment-fid.sql
View File

@@ -1,6 +1,14 @@
-- RAG metadata migration (MySQL 8). Safe to execute repeatedly.
-- 注意：MySQL 8 不支持 ALTER TABLE ... ADD COLUMN IF NOT EXISTS（仅 MariaDB 支持），
-- 因此列的增量添加统一用 information_schema 守卫 + PREPARE 实现幂等。
SET @add_file_hash_col = IF(EXISTS(
SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE()
AND table_name = 'knowledge_attach' AND column_name = 'file_hash'),
'SELECT 1',
'ALTER TABLE `knowledge_attach` ADD COLUMN `file_hash` varchar(64) NULL DEFAULT NULL COMMENT ''文件SHA-256摘要'' AFTER `doc_id`');
PREPARE stmt FROM @add_file_hash_col; EXECUTE stmt; DEALLOCATE PREPARE stmt;
ALTER TABLE `knowledge_attach`
ADD COLUMN IF NOT EXISTS `file_hash` varchar(64) NULL DEFAULT NULL COMMENT '文件SHA-256摘要' AFTER `doc_id`,
MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID';
SET @add_file_hash = IF(EXISTS(
@@ -9,8 +17,14 @@ SET @add_file_hash = IF(EXISTS(
'SELECT 1', 'ALTER TABLE `knowledge_attach` ADD UNIQUE INDEX `uk_knowledge_file_hash` (`knowledge_id`, `file_hash`)');
PREPARE stmt FROM @add_file_hash; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @add_fid_col = IF(EXISTS(
SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE()
AND table_name = 'knowledge_fragment' AND column_name = 'fid'),
'SELECT 1',
'ALTER TABLE `knowledge_fragment` ADD COLUMN `fid` varchar(32) NULL DEFAULT NULL COMMENT ''向量库片段ID'' AFTER `id`');
PREPARE stmt FROM @add_fid_col; EXECUTE stmt; DEALLOCATE PREPARE stmt;
ALTER TABLE `knowledge_fragment`
ADD COLUMN IF NOT EXISTS `fid` varchar(32) NULL DEFAULT NULL COMMENT '向量库片段ID' AFTER `id`,
MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID';
UPDATE `knowledge_fragment`

11
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java
View File

@@ -215,8 +215,6 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
? DocumentSplitConfig.DEFAULT_OVERLAP : knowledgeInfoVo.getOverlapChar().intValue();
DocumentSplitConfig splitConfig = new DocumentSplitConfig(
knowledgeInfoVo.getSeparator(), blockSize, overlap, attach.getType());
List<KnowledgeFragment> oldFragments = knowledgeFragmentMapper.selectList(
Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
// 获取文件信息并下载
List<OssDTO> ossDTOs = ossService.selectByIds(String.valueOf(attach.getOssId()));
@@ -262,12 +260,11 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
try {
// 写入新向量前，先按 docId 清理该文档的旧向量：
// 历史数据的片段 fid 为迁移脚本回填的 MD5 值，与向量库中实际存储的 fid 不一致，
// 按 fid 删除无法命中旧向量，会导致重复向量累积；按 docId 清理对三种向量库均一致有效。
vectorStoreService.removeByDocId(docId, String.valueOf(knowledgeId));
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
for (KnowledgeFragment old : oldFragments) {
if (StringUtils.isNotBlank(old.getFid())) {
vectorStoreService.removeByFid(old.getFid(), String.valueOf(knowledgeId));
}
}
} catch (Exception vectorError) {
for (String newFid : fids) {
try {