mirror of
https://gitcode.com/ageerle/ruoyi-ai.git
synced 2026-09-15 17:35:00 +00:00
fix(rag): 合并后修正-升级脚本兼容 MySQL 8、重解析按 docId 清理旧向量
1. 升级脚本 2026-07-20-knowledge-fragment-fid.sql 使用了 MySQL 8 不支持的 ADD COLUMN IF NOT EXISTS(MariaDB 语法),在存量库执行会直接语法报错。 改为与脚本内索引守卫一致的 information_schema 判断 + PREPARE 方式,保持幂等。 2. parse() 重新解析时按 fragment.fid 删除旧向量:存量数据的 fid 为迁移脚本 回填的 MD5 值,与向量库中实际存储的随机 fid 不一致,首次重解析无法命中 旧向量,导致新旧向量重复累积。改为写入新向量前先按 docId 清理(三种 向量库策略均支持),写入失败时仍补偿删除新 fid。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,14 @@
|
|||||||
-- RAG metadata migration (MySQL 8). Safe to execute repeatedly.
|
-- RAG metadata migration (MySQL 8). Safe to execute repeatedly.
|
||||||
|
-- 注意:MySQL 8 不支持 ALTER TABLE ... ADD COLUMN IF NOT EXISTS(仅 MariaDB 支持),
|
||||||
|
-- 因此列的增量添加统一用 information_schema 守卫 + PREPARE 实现幂等。
|
||||||
|
SET @add_file_hash_col = IF(EXISTS(
|
||||||
|
SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE()
|
||||||
|
AND table_name = 'knowledge_attach' AND column_name = 'file_hash'),
|
||||||
|
'SELECT 1',
|
||||||
|
'ALTER TABLE `knowledge_attach` ADD COLUMN `file_hash` varchar(64) NULL DEFAULT NULL COMMENT ''文件SHA-256摘要'' AFTER `doc_id`');
|
||||||
|
PREPARE stmt FROM @add_file_hash_col; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
ALTER TABLE `knowledge_attach`
|
ALTER TABLE `knowledge_attach`
|
||||||
ADD COLUMN IF NOT EXISTS `file_hash` varchar(64) NULL DEFAULT NULL COMMENT '文件SHA-256摘要' AFTER `doc_id`,
|
|
||||||
MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID';
|
MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID';
|
||||||
|
|
||||||
SET @add_file_hash = IF(EXISTS(
|
SET @add_file_hash = IF(EXISTS(
|
||||||
@@ -9,8 +17,14 @@ SET @add_file_hash = IF(EXISTS(
|
|||||||
'SELECT 1', 'ALTER TABLE `knowledge_attach` ADD UNIQUE INDEX `uk_knowledge_file_hash` (`knowledge_id`, `file_hash`)');
|
'SELECT 1', 'ALTER TABLE `knowledge_attach` ADD UNIQUE INDEX `uk_knowledge_file_hash` (`knowledge_id`, `file_hash`)');
|
||||||
PREPARE stmt FROM @add_file_hash; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
PREPARE stmt FROM @add_file_hash; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @add_fid_col = IF(EXISTS(
|
||||||
|
SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE()
|
||||||
|
AND table_name = 'knowledge_fragment' AND column_name = 'fid'),
|
||||||
|
'SELECT 1',
|
||||||
|
'ALTER TABLE `knowledge_fragment` ADD COLUMN `fid` varchar(32) NULL DEFAULT NULL COMMENT ''向量库片段ID'' AFTER `id`');
|
||||||
|
PREPARE stmt FROM @add_fid_col; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
ALTER TABLE `knowledge_fragment`
|
ALTER TABLE `knowledge_fragment`
|
||||||
ADD COLUMN IF NOT EXISTS `fid` varchar(32) NULL DEFAULT NULL COMMENT '向量库片段ID' AFTER `id`,
|
|
||||||
MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID';
|
MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID';
|
||||||
|
|
||||||
UPDATE `knowledge_fragment`
|
UPDATE `knowledge_fragment`
|
||||||
|
|||||||
@@ -215,8 +215,6 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
|||||||
? DocumentSplitConfig.DEFAULT_OVERLAP : knowledgeInfoVo.getOverlapChar().intValue();
|
? DocumentSplitConfig.DEFAULT_OVERLAP : knowledgeInfoVo.getOverlapChar().intValue();
|
||||||
DocumentSplitConfig splitConfig = new DocumentSplitConfig(
|
DocumentSplitConfig splitConfig = new DocumentSplitConfig(
|
||||||
knowledgeInfoVo.getSeparator(), blockSize, overlap, attach.getType());
|
knowledgeInfoVo.getSeparator(), blockSize, overlap, attach.getType());
|
||||||
List<KnowledgeFragment> oldFragments = knowledgeFragmentMapper.selectList(
|
|
||||||
Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
|
|
||||||
|
|
||||||
// 获取文件信息并下载
|
// 获取文件信息并下载
|
||||||
List<OssDTO> ossDTOs = ossService.selectByIds(String.valueOf(attach.getOssId()));
|
List<OssDTO> ossDTOs = ossService.selectByIds(String.valueOf(attach.getOssId()));
|
||||||
@@ -262,12 +260,11 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
|||||||
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
|
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
|
||||||
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
|
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
|
||||||
try {
|
try {
|
||||||
|
// 写入新向量前,先按 docId 清理该文档的旧向量:
|
||||||
|
// 历史数据的片段 fid 为迁移脚本回填的 MD5 值,与向量库中实际存储的 fid 不一致,
|
||||||
|
// 按 fid 删除无法命中旧向量,会导致重复向量累积;按 docId 清理对三种向量库均一致有效。
|
||||||
|
vectorStoreService.removeByDocId(docId, String.valueOf(knowledgeId));
|
||||||
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
|
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
|
||||||
for (KnowledgeFragment old : oldFragments) {
|
|
||||||
if (StringUtils.isNotBlank(old.getFid())) {
|
|
||||||
vectorStoreService.removeByFid(old.getFid(), String.valueOf(knowledgeId));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (Exception vectorError) {
|
} catch (Exception vectorError) {
|
||||||
for (String newFid : fids) {
|
for (String newFid : fids) {
|
||||||
try {
|
try {
|
||||||
|
|||||||
Reference in New Issue
Block a user