mirror of
https://gitcode.com/ageerle/ruoyi-ai.git
synced 2026-09-13 00:14:59 +00:00
Compare commits
18 Commits
24bee53f9e
...
9adeee6c83
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9adeee6c83 | ||
|
|
e42d35f3bf | ||
|
|
a69d1f51b9 | ||
|
|
911ae6cb1f | ||
|
|
9a2f326e42 | ||
|
|
46a8d6b552 | ||
|
|
cf5423f69f | ||
|
|
177375a7b7 | ||
|
|
016bdbdd94 | ||
|
|
41e0dcb528 | ||
|
|
787e447d09 | ||
|
|
9c092ae3bb | ||
|
|
42bc8fea95 | ||
|
|
c4fc1e6fcf | ||
|
|
6f6d0893ec | ||
|
|
a6a55202a3 | ||
|
|
97ce15d116 | ||
|
|
879fe48945 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -53,3 +53,4 @@ logs/
|
||||
|
||||
.flattened-pom.xml
|
||||
/.claude/settings.local.json
|
||||
/docs/docker/milvus/volumes/
|
||||
|
||||
64
RAG_TEST_REPORT_2026-07-20.md
Normal file
64
RAG_TEST_REPORT_2026-07-20.md
Normal file
@@ -0,0 +1,64 @@
|
||||
# RAG 完整修复与全量验收报告
|
||||
|
||||
验收时间:2026-07-21(Asia/Shanghai)
|
||||
验收对象:当前未提交工作区(保留原有改动)
|
||||
|
||||
## 结论
|
||||
|
||||
计划内的 1–15 项工程缺陷已完成代码修复,默认 Maven 构建已从“跳过测试”改为真实执行测试。全仓 37 个 reactor 模块测试成功,`ruoyi-chat` 49/49 通过,两个前端生产构建通过,`git diff --check` 通过。
|
||||
|
||||
本机已运行 MySQL、Redis、MinIO 和 Weaviate 1.30.0;Milvus/Qdrant 容器以及有效的 embedding/chat/rerank provider 凭证不存在,因此这三项真实 provider/存储引擎冒烟被标记为环境限制,不影响确定性代码验收。
|
||||
|
||||
## 1–15 项验收
|
||||
|
||||
| # | 状态 | 修复/证据 |
|
||||
|---|---|---|
|
||||
| 1 | 通过 | Markdown/Java/字符分片的边界、空文档、超长块回归通过。 |
|
||||
| 2 | 通过 | Supervisor 每轮仅保留一个 RAG 入口,不再用已含 RAG 的 prompt 重复检索。 |
|
||||
| 3 | 通过 | 历史消息进入 Supervisor prompt,检索 query 与最终 prompt 分离。 |
|
||||
| 4 | 通过 | `fid` 稳定 ID 贯穿 DB/三种向量库/RRF,融合去重回归通过。 |
|
||||
| 5 | 通过 | aiflow vector/hybrid 复用统一检索服务;graph 明确返回不支持,不再伪装为 vector。 |
|
||||
| 6 | 通过 | 重解析改为先写新 fid、再清旧向量、最后替换 DB;失败补偿新向量;删片段/附件/库遇向量删除失败即中止。 |
|
||||
| 7 | 通过 | embedding/rerank provider 使用 prototype 实例,工厂缓存可按模型刷新,避免跨配置污染。 |
|
||||
| 8 | 通过 | `similarityThreshold` 仅用于粗召回;`rerankScoreThreshold` 仅在 rerank 真实成功后生效,回归测试通过。 |
|
||||
| 9 | 通过 | 默认配置和 Compose 统一为 Weaviate 1.30.0、`28080:8080`。 |
|
||||
| 10 | 通过 | 三种策略均使用 `embedAll`;Weaviate batch objects、Milvus `addAll`、Qdrant `addAll`。 |
|
||||
| 11 | 通过 | upload/parse/retrieval 权限保留,parse/retrieval 增加分布式防重复提交,upload 由现有知识库+文件名唯一约束兜底。 |
|
||||
| 12 | 通过 | 分隔符使用字面量语义,`|`/`.`/`*` 回归通过。 |
|
||||
| 13 | 通过 | hybrid 通道失败可降级到 vector;所有可用通道都失败时抛出明确业务异常。 |
|
||||
| 14 | 通过 | Weaviate client 稳定懒加载单例;schema 仅在已存在或创建成功后进入缓存。 |
|
||||
| 15 | 通过 | 工厂新增严格 `getStrategy(type)`,知识库 `vectorModel` 优先,空值才回退全局,非法值直接报错。 |
|
||||
|
||||
## 其他完成项
|
||||
|
||||
- 多知识库并行检索,按 `kid + docId + fid` 去重,统一上限和字符预算。
|
||||
- 5 分钟短 TTL 检索缓存,key 覆盖检索参数,知识数据变更主动失效。
|
||||
- rerank 仅保留 provider 实际返回的文档。
|
||||
- 知识库文档数改为 group-by 查询,消除该 N+1。
|
||||
- Milvus/Qdrant/Weaviate 的删 collection/doc/fid 语义对齐;Milvus 删库改为 drop collection。
|
||||
- MCP `npx` 根据操作系统解析,支持系统属性/环境变量覆盖。
|
||||
- `fid` 非空唯一、`doc_id varchar(32)`、租户/用户索引与可重复执行迁移脚本已提供。
|
||||
- 用户端聊天页已接入知识库列表和最小选择器。
|
||||
|
||||
## 测试记录
|
||||
|
||||
| 检查 | 结果 |
|
||||
|---|---|
|
||||
| `mvn -Pdev test` | 37/37 reactor 模块 SUCCESS;`ruoyi-chat` 49/49 |
|
||||
| `mvn -Pdev -pl ruoyi-modules/ruoyi-aiflow -am -DskipTests compile` | 21/21 SUCCESS |
|
||||
| `ruoyi-web: pnpm build` | SUCCESS,2621 modules transformed |
|
||||
| `ruoyi-admin: pnpm build` | SUCCESS,10/10 build tasks |
|
||||
| `git diff --check` | SUCCESS,无空白错误 |
|
||||
| Weaviate/MySQL/Redis/MinIO | Docker 服务运行,Weaviate 1.30.0 映射 28080 |
|
||||
| 三向量库 Docker 集成 | SUCCESS;Weaviate 1.30.0、Milvus 2.5.7、Qdrant 1.17.0 真实写入/检索/删除测试 3/3 通过 |
|
||||
| 真实 embedding/chat/rerank | 环境限制:当前配置为无效/占位凭证 |
|
||||
|
||||
本轮未创建新的 `codex_rag_verify_` 持久化数据;上一轮验收数据已清理,未动现有非测试数据。
|
||||
|
||||
## 2026-07-21 三向量库 Docker 补充验收
|
||||
|
||||
- 启动并保留 `ruoyi-rag-milvus`、`ruoyi-rag-milvus-etcd`、`ruoyi-rag-milvus-minio`、`ruoyi-rag-qdrant`,四个容器健康检查均为 `healthy`。
|
||||
- Milvus 专用 MinIO 仅在 Docker 内网可达,没有占用宿主机 9000/9001;Milvus 映射 19530/9091,Qdrant 映射 6333/6334。
|
||||
- `ThreeVectorStoresDockerIT` 使用 32 维确定性 embedding,对三库逐一验证 batch write、vector search、fid delete、docId delete 和 drop collection,3/3 通过。
|
||||
- 首轮测试发现 Milvus `autoFlush=false` 导致批量写入后不可立即检索、元数据删除不可立即见;改为写入和删除返回前 flush 后通过。
|
||||
- 清理后 Weaviate/Qdrant 的 `CodexRagVerify*` collection 计数均为 0,Milvus collection 也由测试 finally 成功 drop;本轮未写入 MySQL 或 OSS 测试数据。
|
||||
@@ -1,8 +1,6 @@
|
||||
version: '3.5'
|
||||
|
||||
services:
|
||||
etcd:
|
||||
container_name: milvus-etcd
|
||||
container_name: ruoyi-rag-milvus-etcd
|
||||
image: quay.io/coreos/etcd:v3.5.18
|
||||
environment:
|
||||
- ETCD_AUTO_COMPACTION_MODE=revision
|
||||
@@ -19,14 +17,11 @@ services:
|
||||
retries: 3
|
||||
|
||||
minio:
|
||||
container_name: milvus-minio
|
||||
container_name: ruoyi-rag-milvus-minio
|
||||
image: minio/minio:RELEASE.2023-03-20T20-16-18Z
|
||||
environment:
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
ports:
|
||||
- "9001:9001"
|
||||
- "9000:9000"
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
|
||||
command: minio server /minio_data --console-address ":9001"
|
||||
@@ -37,7 +32,7 @@ services:
|
||||
retries: 3
|
||||
|
||||
standalone:
|
||||
container_name: milvus-standalone
|
||||
container_name: ruoyi-rag-milvus
|
||||
image: milvusdb/milvus:v2.5.7
|
||||
command: ["milvus", "run", "standalone"]
|
||||
security_opt:
|
||||
@@ -61,7 +56,7 @@ services:
|
||||
- "minio"
|
||||
|
||||
attu:
|
||||
container_name: attu
|
||||
container_name: ruoyi-rag-attu
|
||||
image: zilliz/attu:v2.5.7
|
||||
environment:
|
||||
MILVUS_URL: milvus-standalone:19530
|
||||
@@ -72,4 +67,4 @@ services:
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: milvus
|
||||
name: ruoyi-rag-milvus
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
---
|
||||
services:
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
container_name: ruoyi-rag-qdrant
|
||||
image: qdrant/qdrant:v1.17.0
|
||||
ports:
|
||||
- 6333:6333
|
||||
- 6334:6334
|
||||
volumes:
|
||||
- qdrant_data:/qdrant/storage
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/6333 && printf \"GET /healthz HTTP/1.1\\r\\nHost: localhost\\r\\nConnection: close\\r\\n\\r\\n\" >&3 && grep -q \"200 OK\" <&3'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
qdrant_data:
|
||||
...
|
||||
|
||||
@@ -5,12 +5,12 @@ services:
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- '6038'
|
||||
- '8080'
|
||||
- --scheme
|
||||
- http
|
||||
image: semitechnologies/weaviate:1.19.7
|
||||
image: semitechnologies/weaviate:1.30.0
|
||||
ports:
|
||||
- 6038:6038
|
||||
- 28080:8080
|
||||
- 50051:50051
|
||||
volumes:
|
||||
- weaviate_data:/var/lib/weaviate
|
||||
|
||||
@@ -126,7 +126,7 @@ CREATE TABLE `chat_provider` (
|
||||
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`provider_name` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '厂商名称',
|
||||
`provider_code` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '厂商编码',
|
||||
`provider_icon` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '厂商图标',
|
||||
`provider_icon` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '厂商图标',
|
||||
`provider_desc` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '厂商描述',
|
||||
`api_host` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'API地址',
|
||||
`status` char(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT '0' COMMENT '状态(0正常 1停用)',
|
||||
@@ -1131,7 +1131,8 @@ CREATE TABLE `knowledge_attach` (
|
||||
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`knowledge_id` bigint NOT NULL COMMENT '知识库ID',
|
||||
`oss_id` bigint NULL DEFAULT NULL COMMENT '对象存储ID',
|
||||
`doc_id` varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '文档ID',
|
||||
`doc_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '文档ID',
|
||||
`file_hash` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '文件SHA-256摘要',
|
||||
`name` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '附件名称',
|
||||
`type` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '附件类型',
|
||||
`create_dept` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '部门',
|
||||
@@ -1143,7 +1144,8 @@ CREATE TABLE `knowledge_attach` (
|
||||
`tenant_id` bigint NOT NULL DEFAULT 0 COMMENT '租户Id',
|
||||
`status` tinyint NULL DEFAULT 0 COMMENT '解析状态: 0待解析, 1解析中, 2已解析, 3解析失败',
|
||||
PRIMARY KEY (`id`) USING BTREE,
|
||||
UNIQUE INDEX `idx_kname`(`knowledge_id` ASC, `name` ASC) USING BTREE
|
||||
UNIQUE INDEX `idx_kname`(`knowledge_id` ASC, `name` ASC) USING BTREE,
|
||||
UNIQUE INDEX `uk_knowledge_file_hash`(`knowledge_id`, `file_hash`) USING BTREE
|
||||
) ENGINE = InnoDB AUTO_INCREMENT = 2033199209203183619 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '知识库附件' ROW_FORMAT = DYNAMIC;
|
||||
|
||||
-- ----------------------------
|
||||
@@ -1156,8 +1158,9 @@ CREATE TABLE `knowledge_attach` (
|
||||
DROP TABLE IF EXISTS `knowledge_fragment`;
|
||||
CREATE TABLE `knowledge_fragment` (
|
||||
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`fid` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '向量库片段ID',
|
||||
`idx` int NOT NULL COMMENT '片段索引下标',
|
||||
`doc_id` varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '文档ID',
|
||||
`doc_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '文档ID',
|
||||
`content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '文档内容',
|
||||
`create_dept` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '部门',
|
||||
`create_by` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '创建人',
|
||||
@@ -1168,6 +1171,9 @@ CREATE TABLE `knowledge_fragment` (
|
||||
`tenant_id` bigint NOT NULL DEFAULT 0 COMMENT '租户Id',
|
||||
`knowledge_id` bigint NULL DEFAULT NULL COMMENT '知识库ID',
|
||||
PRIMARY KEY (`id`) USING BTREE,
|
||||
UNIQUE INDEX `uk_fid`(`fid`) USING BTREE,
|
||||
INDEX `idx_doc_id`(`doc_id`) USING BTREE,
|
||||
INDEX `idx_knowledge_id`(`knowledge_id`) USING BTREE,
|
||||
FULLTEXT INDEX `ft_content`(`content`) WITH PARSER `ngram`
|
||||
) ENGINE = InnoDB AUTO_INCREMENT = 2033199209131880451 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '知识片段' ROW_FORMAT = DYNAMIC;
|
||||
|
||||
@@ -1206,7 +1212,9 @@ CREATE TABLE `knowledge_info` (
|
||||
`enable_hybrid` tinyint(1) NULL DEFAULT 0 COMMENT '是否启用混合检索',
|
||||
`hybrid_alpha` double NULL DEFAULT 0.5 COMMENT '混合检索权重比例 (0.0=纯向量, 1.0=纯关键词)',
|
||||
`system_prompt` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '系统提示词',
|
||||
PRIMARY KEY (`id`) USING BTREE
|
||||
PRIMARY KEY (`id`) USING BTREE,
|
||||
INDEX `idx_tenant_user` (`tenant_id`, `user_id`) USING BTREE,
|
||||
INDEX `idx_tenant_share` (`tenant_id`, `share`) USING BTREE
|
||||
) ENGINE = InnoDB AUTO_INCREMENT = 2033198818050781187 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '知识库' ROW_FORMAT = DYNAMIC;
|
||||
|
||||
-- ----------------------------
|
||||
@@ -2279,6 +2287,15 @@ INSERT INTO `sys_config` VALUES (2018858143803641859, '154726', '用户管理-
|
||||
INSERT INTO `sys_config` VALUES (2018858143803641860, '154726', '主框架页-侧边栏主题', 'sys.index.sideTheme', 'theme-dark', 'Y', 103, 1, '2026-02-04 09:24:25', 1, '2026-02-04 09:24:25', '深色主题theme-dark,浅色主题theme-light');
|
||||
INSERT INTO `sys_config` VALUES (2018858143803641861, '154726', '账号自助-是否开启用户注册功能', 'sys.account.registerUser', 'false', 'Y', 103, 1, '2026-02-04 09:24:25', 1, '2026-02-04 09:24:25', '是否开启注册用户功能(true开启,false关闭)');
|
||||
INSERT INTO `sys_config` VALUES (2018858143803641862, '154726', 'OSS预览列表资源开关', 'sys.oss.previewListResource', 'true', 'Y', 103, 1, '2026-02-04 09:24:25', 1, '2026-02-04 09:24:25', 'true:开启, false:关闭');
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (2027192921483309058, '000000', 'HTTP请求节点响应模板', 'node.httpRequest.template', '✅ HTTP请求节点:结束响应 - ', 'Y', 103, 1, '2026-02-27 09:23:51', 1, '2026-02-27 09:31:41', NULL);
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (2027193296990957569, '000000', '文生图节点响应模板', 'node.image.template', '🎨 文生图节点:结束响应 - 图片URL: ', 'Y', 103, 1, '2026-02-27 09:25:20', 1, '2026-02-27 09:31:52', NULL);
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (2027193820393959425, '000000', '发送邮箱节点响应模板', 'node.mailsend.template', '📧 发送邮箱节点:结束响应 - ', 'Y', 103, 1, '2026-02-27 09:27:25', 1, '2026-02-27 09:32:05', NULL);
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (2027194134438277122, '000000', '结束节点响应模板', 'node.end.template', '🔚 流程已执行完毕,如果您有其他需求,请随时重新发起请求。', 'Y', 103, 1, '2026-02-27 09:28:40', 1, '2026-02-27 09:32:53', NULL);
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (2027206492573335554, '000000', '人机交互节点响应模板', 'node.humanFeedback.template', '👤 人机交互节点:等待用户操作 - ', 'Y', 103, 1, '2026-02-27 10:17:46', 1, '2026-02-27 10:17:46', NULL);
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (2027208880369647617, '000000', '条件分支节点响应模板', 'node.switch.template', '🔀 条件分支节点:触发 -> 跳转到节点 ', 'Y', 103, 1, '2026-02-27 10:27:15', 1, '2026-02-27 10:35:54', NULL);
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (2027213914603995137, '000000', '大模型回答节点响应模板', 'node.llmAnswer.template', '🤖 LLM 节点 生成回答:', 'Y', 103, 1, '2026-02-27 10:47:16', 1, '2026-02-27 10:52:40', NULL);
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (2027214387000066050, '000000', '关键词提取响应模板', 'node.keywordExtractor.template', '🔑 关键词提取节点 处理完成 : ', 'Y', 103, 1, '2026-02-27 10:49:08', 1, '2026-02-27 10:52:08', NULL);
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (2027217577397391361, '000000', '工作流异常响应模板', 'node.exception.template', '🛑 工作流发生异常:', 'N', 103, 1, '2026-02-27 11:01:49', 1, '2026-02-27 11:02:01', NULL);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for sys_dept
|
||||
|
||||
58
docs/script/sql/update/2026-07-20-knowledge-fragment-fid.sql
Normal file
58
docs/script/sql/update/2026-07-20-knowledge-fragment-fid.sql
Normal file
@@ -0,0 +1,58 @@
|
||||
-- RAG metadata migration (MySQL 8). Safe to execute repeatedly.
|
||||
-- 注意:MySQL 8 不支持 ALTER TABLE ... ADD COLUMN IF NOT EXISTS(仅 MariaDB 支持),
|
||||
-- 因此列的增量添加统一用 information_schema 守卫 + PREPARE 实现幂等。
|
||||
SET @add_file_hash_col = IF(EXISTS(
|
||||
SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE()
|
||||
AND table_name = 'knowledge_attach' AND column_name = 'file_hash'),
|
||||
'SELECT 1',
|
||||
'ALTER TABLE `knowledge_attach` ADD COLUMN `file_hash` varchar(64) NULL DEFAULT NULL COMMENT ''文件SHA-256摘要'' AFTER `doc_id`');
|
||||
PREPARE stmt FROM @add_file_hash_col; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
ALTER TABLE `knowledge_attach`
|
||||
MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID';
|
||||
|
||||
SET @add_file_hash = IF(EXISTS(
|
||||
SELECT 1 FROM information_schema.statistics WHERE table_schema = DATABASE()
|
||||
AND table_name = 'knowledge_attach' AND index_name = 'uk_knowledge_file_hash'),
|
||||
'SELECT 1', 'ALTER TABLE `knowledge_attach` ADD UNIQUE INDEX `uk_knowledge_file_hash` (`knowledge_id`, `file_hash`)');
|
||||
PREPARE stmt FROM @add_file_hash; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
SET @add_fid_col = IF(EXISTS(
|
||||
SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE()
|
||||
AND table_name = 'knowledge_fragment' AND column_name = 'fid'),
|
||||
'SELECT 1',
|
||||
'ALTER TABLE `knowledge_fragment` ADD COLUMN `fid` varchar(32) NULL DEFAULT NULL COMMENT ''向量库片段ID'' AFTER `id`');
|
||||
PREPARE stmt FROM @add_fid_col; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
ALTER TABLE `knowledge_fragment`
|
||||
MODIFY COLUMN `doc_id` varchar(32) NULL DEFAULT NULL COMMENT '文档ID';
|
||||
|
||||
UPDATE `knowledge_fragment`
|
||||
SET `fid` = LOWER(MD5(CONCAT('knowledge_fragment:', `id`)))
|
||||
WHERE `fid` IS NULL OR `fid` = '';
|
||||
|
||||
SET @drop_idx_fid = IF(EXISTS(
|
||||
SELECT 1 FROM information_schema.statistics WHERE table_schema = DATABASE()
|
||||
AND table_name = 'knowledge_fragment' AND index_name = 'idx_fid'),
|
||||
'ALTER TABLE `knowledge_fragment` DROP INDEX `idx_fid`', 'SELECT 1');
|
||||
PREPARE stmt FROM @drop_idx_fid; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
SET @add_uk_fid = IF(EXISTS(
|
||||
SELECT 1 FROM information_schema.statistics WHERE table_schema = DATABASE()
|
||||
AND table_name = 'knowledge_fragment' AND index_name = 'uk_fid'),
|
||||
'SELECT 1', 'ALTER TABLE `knowledge_fragment` ADD UNIQUE INDEX `uk_fid` (`fid`)');
|
||||
PREPARE stmt FROM @add_uk_fid; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
ALTER TABLE `knowledge_fragment` MODIFY COLUMN `fid` varchar(32) NOT NULL COMMENT '向量库片段ID';
|
||||
|
||||
SET @add_tenant_user = IF(EXISTS(
|
||||
SELECT 1 FROM information_schema.statistics WHERE table_schema = DATABASE()
|
||||
AND table_name = 'knowledge_info' AND index_name = 'idx_tenant_user'),
|
||||
'SELECT 1', 'ALTER TABLE `knowledge_info` ADD INDEX `idx_tenant_user` (`tenant_id`, `user_id`)');
|
||||
PREPARE stmt FROM @add_tenant_user; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
SET @add_tenant_share = IF(EXISTS(
|
||||
SELECT 1 FROM information_schema.statistics WHERE table_schema = DATABASE()
|
||||
AND table_name = 'knowledge_info' AND index_name = 'idx_tenant_share'),
|
||||
'SELECT 1', 'ALTER TABLE `knowledge_info` ADD INDEX `idx_tenant_share` (`tenant_id`, `share`)');
|
||||
PREPARE stmt FROM @add_tenant_share; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- 加宽 chat_provider.provider_icon 字段 (对应 issue IHPUDA)
|
||||
-- 背景:文件系统使用 minio 私有桶时,厂商图标存的是带签名的临时访问 URL,
|
||||
-- 长度常超过 255,导致「Data too long for column 'provider_icon'」。
|
||||
-- MODIFY COLUMN 可重复执行。
|
||||
|
||||
ALTER TABLE `chat_provider`
|
||||
MODIFY COLUMN `provider_icon` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '厂商图标';
|
||||
@@ -0,0 +1,41 @@
|
||||
-- 补充工作流节点消息模板配置 (对应 issue IJX5VV)
|
||||
-- 背景:NodeMessageTemplateEnum 依赖以下 9 个 sys_config 键,缺失时
|
||||
-- WorkflowMessageUtil.getNodeMessageTemplate 会抛出「请先配置该节点的响应模板」。
|
||||
-- 这批配置在历史提交 20d531c0 中存在,SQL 脚本合并重命名时遗失,此处恢复。
|
||||
-- 幂等:按 config_key + tenant_id 判重,可重复执行。
|
||||
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`)
|
||||
SELECT 2027192921483309058, '000000', 'HTTP请求节点响应模板', 'node.httpRequest.template', '✅ HTTP请求节点:结束响应 - ', 'Y', 103, 1, '2026-02-27 09:23:51', 1, '2026-02-27 09:31:41', NULL
|
||||
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `sys_config` WHERE `config_key` = 'node.httpRequest.template' AND `tenant_id` = '000000');
|
||||
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`)
|
||||
SELECT 2027193296990957569, '000000', '文生图节点响应模板', 'node.image.template', '🎨 文生图节点:结束响应 - 图片URL: ', 'Y', 103, 1, '2026-02-27 09:25:20', 1, '2026-02-27 09:31:52', NULL
|
||||
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `sys_config` WHERE `config_key` = 'node.image.template' AND `tenant_id` = '000000');
|
||||
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`)
|
||||
SELECT 2027193820393959425, '000000', '发送邮箱节点响应模板', 'node.mailsend.template', '📧 发送邮箱节点:结束响应 - ', 'Y', 103, 1, '2026-02-27 09:27:25', 1, '2026-02-27 09:32:05', NULL
|
||||
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `sys_config` WHERE `config_key` = 'node.mailsend.template' AND `tenant_id` = '000000');
|
||||
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`)
|
||||
SELECT 2027194134438277122, '000000', '结束节点响应模板', 'node.end.template', '🔚 流程已执行完毕,如果您有其他需求,请随时重新发起请求。', 'Y', 103, 1, '2026-02-27 09:28:40', 1, '2026-02-27 09:32:53', NULL
|
||||
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `sys_config` WHERE `config_key` = 'node.end.template' AND `tenant_id` = '000000');
|
||||
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`)
|
||||
SELECT 2027206492573335554, '000000', '人机交互节点响应模板', 'node.humanFeedback.template', '👤 人机交互节点:等待用户操作 - ', 'Y', 103, 1, '2026-02-27 10:17:46', 1, '2026-02-27 10:17:46', NULL
|
||||
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `sys_config` WHERE `config_key` = 'node.humanFeedback.template' AND `tenant_id` = '000000');
|
||||
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`)
|
||||
SELECT 2027208880369647617, '000000', '条件分支节点响应模板', 'node.switch.template', '🔀 条件分支节点:触发 -> 跳转到节点 ', 'Y', 103, 1, '2026-02-27 10:27:15', 1, '2026-02-27 10:35:54', NULL
|
||||
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `sys_config` WHERE `config_key` = 'node.switch.template' AND `tenant_id` = '000000');
|
||||
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`)
|
||||
SELECT 2027213914603995137, '000000', '大模型回答节点响应模板', 'node.llmAnswer.template', '🤖 LLM 节点 生成回答:', 'Y', 103, 1, '2026-02-27 10:47:16', 1, '2026-02-27 10:52:40', NULL
|
||||
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `sys_config` WHERE `config_key` = 'node.llmAnswer.template' AND `tenant_id` = '000000');
|
||||
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`)
|
||||
SELECT 2027214387000066050, '000000', '关键词提取响应模板', 'node.keywordExtractor.template', '🔑 关键词提取节点 处理完成 : ', 'Y', 103, 1, '2026-02-27 10:49:08', 1, '2026-02-27 10:52:08', NULL
|
||||
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `sys_config` WHERE `config_key` = 'node.keywordExtractor.template' AND `tenant_id` = '000000');
|
||||
|
||||
INSERT INTO `sys_config` (`config_id`, `tenant_id`, `config_name`, `config_key`, `config_value`, `config_type`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`)
|
||||
SELECT 2027217577397391361, '000000', '工作流异常响应模板', 'node.exception.template', '🛑 工作流发生异常:', 'N', 103, 1, '2026-02-27 11:01:49', 1, '2026-02-27 11:02:01', NULL
|
||||
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `sys_config` WHERE `config_key` = 'node.exception.template' AND `tenant_id` = '000000');
|
||||
2
pom.xml
2
pom.xml
@@ -84,7 +84,7 @@
|
||||
<maven-surefire-plugin.version>3.5.3</maven-surefire-plugin.version>
|
||||
<flatten-maven-plugin.version>1.3.0</flatten-maven-plugin.version>
|
||||
<!-- 打包默认跳过测试 -->
|
||||
<skipTests>true</skipTests>
|
||||
<skipTests>false</skipTests>
|
||||
</properties>
|
||||
|
||||
<profiles>
|
||||
|
||||
@@ -60,7 +60,7 @@ spring:
|
||||
# rewriteBatchedStatements=true 批处理优化 大幅提升批量插入更新删除性能(对数据库有性能损耗 使用批量操作应考虑性能问题)
|
||||
url: jdbc:mysql://127.0.0.1:3306/ruoyi-ai?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
|
||||
username: root
|
||||
password: root
|
||||
password: 123456
|
||||
# agent:
|
||||
# url: jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
|
||||
# # url: jdbc:mysql://localhost:3306/agent_db
|
||||
|
||||
@@ -301,11 +301,12 @@ warm-flow:
|
||||
vector-store:
|
||||
# 向量存储类型 可选(weaviate/milvus/qdrant)
|
||||
# 如需修改向量库类型,请修改此配置值!
|
||||
type: milvus
|
||||
# 注意:需与 docker-compose 实际部署的向量库保持一致(当前 compose 内置 weaviate,映射端口 28080)
|
||||
type: weaviate
|
||||
# Weaviate配置
|
||||
weaviate:
|
||||
protocol: http
|
||||
host: 127.0.0.1:6038
|
||||
host: 127.0.0.1:28080
|
||||
classname: LocalKnowledge
|
||||
# Milvus配置
|
||||
milvus:
|
||||
|
||||
@@ -29,6 +29,12 @@
|
||||
<artifactId>ruoyi-common-chat</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- 复用聊天模块的知识库统一检索能力(向量/混合/重排) -->
|
||||
<dependency>
|
||||
<groupId>org.ruoyi</groupId>
|
||||
<artifactId>ruoyi-chat</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.ruoyi</groupId>
|
||||
<artifactId>ruoyi-common-web</artifactId>
|
||||
|
||||
@@ -27,13 +27,13 @@ public class WorkflowRuntime extends BaseEntity {
|
||||
@TableField("workflow_id")
|
||||
private Long workflowId;
|
||||
|
||||
@TableField(value = "input")
|
||||
@TableField(value = "`input`")
|
||||
private String input;
|
||||
|
||||
@TableField(value = "output")
|
||||
@TableField(value = "`output`")
|
||||
private String output;
|
||||
|
||||
@TableField("status")
|
||||
@TableField("`status`")
|
||||
private Integer status;
|
||||
|
||||
@TableField("status_remark")
|
||||
|
||||
@@ -29,13 +29,13 @@ public class WorkflowRuntimeNode extends BaseEntity {
|
||||
@TableField("node_id")
|
||||
private Long nodeId;
|
||||
|
||||
@TableField(value = "input")
|
||||
@TableField(value = "`input`")
|
||||
private String input;
|
||||
|
||||
@TableField(value = "output")
|
||||
@TableField(value = "`output`")
|
||||
private String output;
|
||||
|
||||
@TableField("status")
|
||||
@TableField("`status`")
|
||||
private Integer status;
|
||||
|
||||
@TableField("status_remark")
|
||||
|
||||
@@ -80,9 +80,10 @@ public class KnowledgeRetrievalNode extends AbstractWfNode {
|
||||
String retrievalResult;
|
||||
String mode = config.getRetrievalMode() != null ? config.getRetrievalMode().toLowerCase() : "vector";
|
||||
|
||||
// 目前只支持向量检索,图谱检索需要依赖graph模块
|
||||
if ("graph".equals(mode) || "hybrid".equals(mode)) {
|
||||
log.warn("Graph retrieval mode is not supported in workflow-api module, falling back to vector retrieval");
|
||||
// 图谱检索需要依赖 graph 模块,暂不支持;vector/hybrid 由统一检索服务处理
|
||||
if ("graph".equals(mode)) {
|
||||
log.warn("Graph retrieval mode is not supported");
|
||||
throw new UnsupportedOperationException("GraphRAG retrieval is not supported");
|
||||
}
|
||||
|
||||
retrievalResult = retrieveFromVector(config, finalQuery);
|
||||
@@ -203,18 +204,75 @@ public class KnowledgeRetrievalNode extends AbstractWfNode {
|
||||
}
|
||||
|
||||
/**
|
||||
* 从向量库检索
|
||||
* 从向量库检索(复用聊天模块的统一检索服务:向量 + 可选混合检索 + 可选重排)
|
||||
*/
|
||||
private String retrieveFromVector(KnowledgeRetrievalNodeConfig config, String query) {
|
||||
try {
|
||||
|
||||
// 获取知识库信息以获取embedding模型配置
|
||||
Long knowledgeId = Long.parseLong(config.getKnowledgeId());
|
||||
|
||||
// 合并结果
|
||||
String mergedResult = "根据知识库id + query 查询知识库内容";
|
||||
org.ruoyi.service.knowledge.IKnowledgeInfoService knowledgeInfoService =
|
||||
SpringUtil.getBean(org.ruoyi.service.knowledge.IKnowledgeInfoService.class);
|
||||
org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo kb = knowledgeInfoService.queryById(knowledgeId);
|
||||
if (kb == null) {
|
||||
log.error("Knowledge base not found: {}", knowledgeId);
|
||||
return "错误:知识库不存在, id=" + knowledgeId;
|
||||
}
|
||||
|
||||
return mergedResult;
|
||||
org.ruoyi.common.chat.service.chat.IChatModelService chatModelService =
|
||||
SpringUtil.getBean(org.ruoyi.common.chat.service.chat.IChatModelService.class);
|
||||
org.ruoyi.common.chat.domain.vo.chat.ChatModelVo embModel =
|
||||
chatModelService.selectModelByName(kb.getEmbeddingModel());
|
||||
if (embModel == null) {
|
||||
log.error("Embedding model not found: {}", kb.getEmbeddingModel());
|
||||
return "错误:知识库未配置有效的向量模型";
|
||||
}
|
||||
|
||||
// 组装检索参数:节点配置优先,混合检索/重排继承知识库配置
|
||||
org.ruoyi.domain.bo.vector.QueryVectorBo bo = new org.ruoyi.domain.bo.vector.QueryVectorBo();
|
||||
bo.setQuery(query);
|
||||
bo.setKid(String.valueOf(knowledgeId));
|
||||
bo.setMaxResults(config.getTopK() != null ? config.getTopK() : kb.getRetrieveLimit());
|
||||
bo.setSimilarityThreshold(config.getSimilarityThreshold() != null
|
||||
? config.getSimilarityThreshold() : kb.getSimilarityThreshold());
|
||||
bo.setEmbeddingModelName(kb.getEmbeddingModel());
|
||||
bo.setVectorModelName(kb.getVectorModel());
|
||||
bo.setApiKey(embModel.getApiKey());
|
||||
bo.setBaseUrl(embModel.getApiHost());
|
||||
|
||||
String mode = config.getRetrievalMode() != null ? config.getRetrievalMode().toLowerCase() : "vector";
|
||||
boolean enableHybrid = "hybrid".equals(mode)
|
||||
|| (kb.getEnableHybrid() != null && kb.getEnableHybrid() == 1);
|
||||
bo.setEnableHybrid(enableHybrid);
|
||||
bo.setHybridAlpha(kb.getHybridAlpha());
|
||||
bo.setEnableRerank(kb.getEnableRerank() != null && kb.getEnableRerank() == 1);
|
||||
bo.setRerankModelName(kb.getRerankModel());
|
||||
bo.setRerankTopN(kb.getRerankTopN());
|
||||
bo.setRerankScoreThreshold(kb.getRerankScoreThreshold());
|
||||
|
||||
org.ruoyi.service.retrieval.KnowledgeRetrievalService retrievalService =
|
||||
SpringUtil.getBean(org.ruoyi.service.retrieval.KnowledgeRetrievalService.class);
|
||||
java.util.List<org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo> results = retrievalService.retrieve(bo);
|
||||
if (results == null || results.isEmpty()) {
|
||||
log.info("Knowledge retrieval returned no results, kid={}, query={}", knowledgeId, query);
|
||||
return "";
|
||||
}
|
||||
|
||||
// 合并结果
|
||||
boolean returnSource = config.getReturnSource() == null || config.getReturnSource();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < results.size(); i++) {
|
||||
org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo vo = results.get(i);
|
||||
sb.append(i + 1).append(". ").append(vo.getContent());
|
||||
if (returnSource && StringUtils.isNotBlank(vo.getSourceName())) {
|
||||
sb.append("(来源: ").append(vo.getSourceName());
|
||||
if (vo.getScore() != null) {
|
||||
sb.append(String.format(", 相关度: %.3f", vo.getScore()));
|
||||
}
|
||||
sb.append(")");
|
||||
}
|
||||
sb.append("\n");
|
||||
}
|
||||
return sb.toString().trim();
|
||||
} catch (NumberFormatException e) {
|
||||
log.error("Invalid knowledge base ID format: {}", config.getKnowledgeId(), e);
|
||||
return "错误:知识库ID格式无效";
|
||||
|
||||
@@ -30,6 +30,9 @@ public interface SqlAgent {
|
||||
- You MUST ALWAYS use queryAllTables first to query all tables in the database before executing any SQL queries
|
||||
- Only after understanding the database schema can you construct and execute appropriate SQL queries
|
||||
- This is mandatory and applies to all queries without exception
|
||||
- If queryAllTables returns NO tables or an empty list, you MUST NOT call executeSql or queryTableSchema
|
||||
- When no tables are available, inform the user: "当前未配置可查询的数据库表,请联系管理员配置"
|
||||
- NEVER attempt to execute any SQL query (including SELECT * FROM xxx) without first confirming available tables
|
||||
""")
|
||||
@UserMessage("""
|
||||
Answer the following question: {{query}}
|
||||
|
||||
@@ -8,9 +8,14 @@ import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.ruoyi.agent.manager.TableSchemaManager;
|
||||
import org.ruoyi.common.core.utils.SpringUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -54,6 +59,22 @@ public class ExecuteSqlQueryTool implements BuiltinToolProvider {
|
||||
return "Error: Only SELECT queries are allowed for security reasons";
|
||||
}
|
||||
|
||||
// 校验表白名单:未配置表时直接拒绝,已配置则校验 SQL 中引用的表
|
||||
TableSchemaManager schemaManager = SpringUtils.getBean(TableSchemaManager.class);
|
||||
List<String> allowedTables = schemaManager.getAllowedTableNames();
|
||||
if (allowedTables.isEmpty()) {
|
||||
return "Error: 当前未配置可查询的数据库表,无法执行任何SQL查询。请联系管理员配置 AGENT_ALLOWED_TABLES";
|
||||
}
|
||||
Set<String> allowedSet = allowedTables.stream()
|
||||
.map(String::toLowerCase)
|
||||
.collect(Collectors.toSet());
|
||||
Set<String> referencedTables = extractTableNames(upperSql);
|
||||
for (String table : referencedTables) {
|
||||
if (!allowedSet.contains(table.toLowerCase())) {
|
||||
return "Error: 表 " + table + " 不在允许查询的表列表中。允许查询的表: " + String.join(", ", allowedTables);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
DataSource dataSource = getDataSource();
|
||||
if (dataSource == null) {
|
||||
@@ -99,6 +120,21 @@ public class ExecuteSqlQueryTool implements BuiltinToolProvider {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 SQL 中提取引用的表名(FROM / JOIN 后的标识符)
|
||||
* 覆盖 FROM t1, t2 / FROM t1 JOIN t2 / FROM `t1` 等常见写法
|
||||
*/
|
||||
private Set<String> extractTableNames(String upperSql) {
|
||||
Set<String> tables = new java.util.HashSet<>();
|
||||
// 匹配 FROM 或 JOIN 后面的表名(支持反引号包裹)
|
||||
Pattern pattern = Pattern.compile("(?:FROM|JOIN)\\s+`?([A-Z0-9_]+)`?", Pattern.CASE_INSENSITIVE);
|
||||
Matcher matcher = pattern.matcher(upperSql);
|
||||
while (matcher.find()) {
|
||||
tables.add(matcher.group(1));
|
||||
}
|
||||
return tables;
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化查询结果
|
||||
* 返回清晰的表格格式,展示关键数据
|
||||
|
||||
@@ -9,6 +9,7 @@ import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeAttachService;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
@@ -106,7 +107,10 @@ public class KnowledgeAttachController extends BaseController {
|
||||
|
||||
/**
|
||||
* 上传知识库附件
|
||||
* 注意:multipart 上传不能加 @RepeatSubmit(其参数序列化不支持 MultipartFile)
|
||||
*/
|
||||
@SaCheckPermission("system:attach:add")
|
||||
@Log(title = "知识库附件", businessType = BusinessType.INSERT)
|
||||
@PostMapping(value = "/upload")
|
||||
public R<String> upload(KnowledgeInfoUploadBo bo){
|
||||
knowledgeAttachService.upload(bo);
|
||||
@@ -118,9 +122,20 @@ public class KnowledgeAttachController extends BaseController {
|
||||
*
|
||||
* @param id 附件ID
|
||||
*/
|
||||
@SaCheckPermission("system:attach:edit")
|
||||
@Log(title = "知识库附件", businessType = BusinessType.UPDATE)
|
||||
@PostMapping("/parse/{id}")
|
||||
@RepeatSubmit()
|
||||
public R<Void> parse(@PathVariable Long id) {
|
||||
knowledgeAttachService.parse(id);
|
||||
return R.ok();
|
||||
}
|
||||
|
||||
@SaCheckPermission("system:attach:edit")
|
||||
@Log(title = "知识库附件批量重新解析", businessType = BusinessType.UPDATE)
|
||||
@PostMapping("/reparse/knowledge/{knowledgeId}")
|
||||
@RepeatSubmit()
|
||||
public R<KnowledgeReparseVo> reparseKnowledge(@PathVariable Long knowledgeId) {
|
||||
return R.ok(knowledgeAttachService.reparseKnowledge(knowledgeId));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +107,9 @@ public class KnowledgeFragmentController extends BaseController {
|
||||
/**
|
||||
* 检索测试
|
||||
*/
|
||||
@SaCheckPermission("system:fragment:list")
|
||||
@PostMapping("/retrieval")
|
||||
@RepeatSubmit()
|
||||
public R<List<KnowledgeRetrievalVo>> retrieval(@RequestBody KnowledgeFragmentBo bo) {
|
||||
return R.ok(knowledgeFragmentService.retrieval(bo));
|
||||
}
|
||||
|
||||
@@ -37,6 +37,9 @@ public class KnowledgeAttach extends BaseEntity {
|
||||
*/
|
||||
private String docId;
|
||||
|
||||
/** SHA-256 content digest used for upload idempotency. */
|
||||
private String fileHash;
|
||||
|
||||
/**
|
||||
* 附件名称
|
||||
*/
|
||||
|
||||
@@ -27,6 +27,11 @@ public class KnowledgeFragment extends BaseEntity {
|
||||
@TableId(value = "id")
|
||||
private Long id;
|
||||
|
||||
/**
|
||||
* 向量库片段ID(与向量库中的 fid 元数据对应,用于向量定位与混合检索融合)
|
||||
*/
|
||||
private String fid;
|
||||
|
||||
/**
|
||||
* 文档ID-用于关联文本块信息
|
||||
*/
|
||||
|
||||
@@ -30,6 +30,11 @@ public class KnowledgeFragmentVo implements Serializable {
|
||||
@ExcelProperty(value = "主键")
|
||||
private Long id;
|
||||
|
||||
/**
|
||||
* 向量库片段ID
|
||||
*/
|
||||
private String fid;
|
||||
|
||||
/**
|
||||
* 文档ID-用于关联文本块信息
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
package org.ruoyi.domain.vo.knowledge;
|
||||
|
||||
public record KnowledgeReparseVo(int submitted, int skipped, int total) {
|
||||
}
|
||||
@@ -25,6 +25,14 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
@Slf4j
|
||||
public class EmbeddingModelFactory {
|
||||
|
||||
/**
|
||||
* 厂商 providerCode → Spring Bean 名称映射
|
||||
* 处理 providerCode 与 @Component 注册名不一致的情况
|
||||
*/
|
||||
private static final Map<String, String> PROVIDER_BEAN_MAPPING = Map.of(
|
||||
"qianwen", "alibailian"
|
||||
);
|
||||
|
||||
private final ApplicationContext applicationContext;
|
||||
|
||||
private final IChatModelService chatModelService;
|
||||
@@ -77,13 +85,22 @@ public class EmbeddingModelFactory {
|
||||
|
||||
/**
|
||||
* 刷新模型缓存
|
||||
* 根据给定的嵌入模型ID从缓存中移除对应的模型
|
||||
* 根据给定的嵌入模型ID解析模型名称后,从缓存中移除对应的模型
|
||||
*
|
||||
* @param embeddingModelId 嵌入模型的唯一标识ID
|
||||
*/
|
||||
public void refreshModel(Long embeddingModelId) {
|
||||
// 从模型缓存中移除指定ID的模型
|
||||
modelCache.remove(embeddingModelId);
|
||||
ChatModelVo modelConfig = chatModelService.queryById(embeddingModelId);
|
||||
if (modelConfig != null) {
|
||||
modelCache.remove(modelConfig.getModelName());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 按模型名称刷新缓存
|
||||
*/
|
||||
public void refreshModelByName(String embeddingModelName) {
|
||||
modelCache.remove(embeddingModelName);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -107,8 +124,10 @@ public class EmbeddingModelFactory {
|
||||
*/
|
||||
private BaseEmbedModelService createModelInstance(String factory, ChatModelVo config) {
|
||||
try {
|
||||
// 解析实际的 Bean 名称(处理 providerCode 与 @Component 注册名不一致的情况)
|
||||
String beanName = PROVIDER_BEAN_MAPPING.getOrDefault(factory, factory);
|
||||
// 从Spring上下文中获取模型实例
|
||||
BaseEmbedModelService model = applicationContext.getBean(factory, BaseEmbedModelService.class);
|
||||
BaseEmbedModelService model = applicationContext.getBean(beanName, BaseEmbedModelService.class);
|
||||
// 配置模型参数
|
||||
model.configure(config);
|
||||
// 增加嵌入模型监听器
|
||||
|
||||
@@ -55,12 +55,22 @@ public class RerankModelFactory {
|
||||
|
||||
/**
|
||||
* 刷新模型缓存
|
||||
* 根据给定的模型ID从缓存中移除对应的模型
|
||||
* 根据给定的模型ID解析模型名称后,从缓存中移除对应的模型
|
||||
*
|
||||
* @param modelId 模型的唯一标识ID
|
||||
*/
|
||||
public void refreshModel(Long modelId) {
|
||||
modelCache.remove(modelId);
|
||||
ChatModelVo modelConfig = chatModelService.queryById(modelId);
|
||||
if (modelConfig != null) {
|
||||
modelCache.remove(modelConfig.getModelName());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 按模型名称刷新缓存
|
||||
*/
|
||||
public void refreshModelByName(String modelName) {
|
||||
modelCache.remove(modelName);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -17,7 +17,7 @@ public class ResourceLoaderFactory {
|
||||
private final ExcelTextSplitter excelTextSplitter;
|
||||
|
||||
public ResourceLoader getLoaderByFileType(String fileType) {
|
||||
fileType = StringUtils.removeStart(fileType, ".");
|
||||
fileType = StringUtils.lowerCase(StringUtils.removeStart(StringUtils.trim(fileType), "."));
|
||||
if (FileTypeConstants.isTextFile(fileType)) {
|
||||
return new TextFileLoader(characterTextSplitter);
|
||||
} else if (FileTypeConstants.isWord(fileType)) {
|
||||
|
||||
@@ -10,6 +10,7 @@ import org.ruoyi.service.vector.impl.MilvusVectorStoreStrategy;
|
||||
import org.ruoyi.service.vector.impl.QdrantVectorStoreStrategy;
|
||||
import org.ruoyi.service.vector.impl.WeaviateVectorStoreStrategy;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.ruoyi.common.core.exception.ServiceException;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
@@ -45,14 +46,20 @@ public class VectorStoreStrategyFactory {
|
||||
* 获取当前配置的向量库策略
|
||||
*/
|
||||
public VectorStoreService getStrategy() {
|
||||
String vectorStoreType = vectorStoreProperties.getType();
|
||||
return getStrategy(null);
|
||||
}
|
||||
|
||||
public VectorStoreService getStrategy(String requestedType) {
|
||||
String vectorStoreType = requestedType;
|
||||
if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) {
|
||||
vectorStoreType = vectorStoreProperties.getType();
|
||||
}
|
||||
if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) {
|
||||
vectorStoreType = "weaviate"; // 默认使用weaviate
|
||||
}
|
||||
VectorStoreService strategy = strategies.get(vectorStoreType.toLowerCase());
|
||||
if (strategy == null) {
|
||||
log.warn("未找到向量库策略: {}, 使用默认策略: weaviate", vectorStoreType);
|
||||
strategy = strategies.get("weaviate");
|
||||
throw new ServiceException("不支持的向量库类型: " + vectorStoreType);
|
||||
}
|
||||
log.debug("使用向量库策略: {}", vectorStoreType);
|
||||
return strategy;
|
||||
|
||||
@@ -6,6 +6,8 @@ import org.apache.ibatis.annotations.Select;
|
||||
import org.ruoyi.domain.entity.knowledge.KnowledgeAttach;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
|
||||
import org.ruoyi.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 知识库附件Mapper接口
|
||||
@@ -21,4 +23,10 @@ public interface KnowledgeAttachMapper extends BaseMapperPlus<KnowledgeAttach, K
|
||||
*/
|
||||
@Select("SELECT COUNT(*) FROM knowledge_attach WHERE knowledge_id = #{knowledgeId}")
|
||||
int countByKnowledgeId(@Param("knowledgeId") Long knowledgeId);
|
||||
|
||||
@Select("<script>SELECT knowledge_id AS knowledgeId, COUNT(*) AS documentCount " +
|
||||
"FROM knowledge_attach WHERE knowledge_id IN " +
|
||||
"<foreach collection='knowledgeIds' item='id' open='(' separator=',' close=')'>#{id}</foreach> " +
|
||||
"GROUP BY knowledge_id</script>")
|
||||
List<Map<String, Object>> countByKnowledgeIds(@Param("knowledgeIds") List<Long> knowledgeIds);
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ public interface KnowledgeFragmentMapper extends BaseMapperPlus<KnowledgeFragmen
|
||||
"</script>")
|
||||
List<DocFragmentCountVo> selectFragmentCountByDocIds(@Param("docIds") List<String> docIds);
|
||||
@Select("<script>" +
|
||||
"SELECT id, doc_id AS docId, content, idx, knowledge_id AS knowledgeId " +
|
||||
"SELECT id, fid, doc_id AS docId, content, idx, knowledge_id AS knowledgeId " +
|
||||
"FROM knowledge_fragment " +
|
||||
"WHERE knowledge_id = #{knowledgeId} " +
|
||||
"AND MATCH (content) AGAINST (#{query} IN NATURAL LANGUAGE MODE) " +
|
||||
|
||||
@@ -17,6 +17,7 @@ import dev.langchain4j.memory.chat.MessageWindowChatMemory;
|
||||
import dev.langchain4j.model.chat.ChatModel;
|
||||
import dev.langchain4j.model.chat.StreamingChatModel;
|
||||
import dev.langchain4j.model.chat.response.ChatResponse;
|
||||
import dev.langchain4j.model.chat.response.PartialThinking;
|
||||
import dev.langchain4j.model.chat.response.StreamingChatResponseHandler;
|
||||
import dev.langchain4j.rag.content.Content;
|
||||
import dev.langchain4j.rag.content.retriever.ContentRetriever;
|
||||
@@ -77,6 +78,7 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@@ -161,7 +163,8 @@ public class ChatServiceFacade implements IChatService {
|
||||
throw new IllegalArgumentException("模型不存在: " + chatRequest.getModel());
|
||||
}
|
||||
|
||||
// 2. 构建上下文消息列表
|
||||
// 2. 构建上下文消息列表(系统提示词 + 历史消息 + 当前用户消息)
|
||||
// 注意:RAG 检索增强统一在 handleAgentChat 中执行一次,此处不再重复检索
|
||||
List<ChatMessage> contextMessages = buildContextMessages(chatRequest, agentVo);
|
||||
|
||||
chatRequest.setEmitter(emitter);
|
||||
@@ -282,12 +285,20 @@ public class ChatServiceFacade implements IChatService {
|
||||
.responseStrategy(SupervisorResponseStrategy.SUMMARY);
|
||||
SupervisorAgent supervisor = supervisorBuilder.build();
|
||||
|
||||
// 知识库增强:智能体绑定了知识库时,对 supervisor 输入做一次 RAG 增强
|
||||
// 知识库增强:智能体绑定了知识库时,对 supervisor 输入做一次 RAG 增强(全程唯一一次检索)
|
||||
String augmentedInput = augmentAgentInput(chatRequest, agentVo);
|
||||
// 智能体自定义系统提示词:supervisor builder 不支持 systemMessage,前置到输入
|
||||
String prompt = (agentVo != null && StringUtils.isNotBlank(agentVo.getSystemPrompt()))
|
||||
? agentVo.getSystemPrompt() + "\n\n" + augmentedInput
|
||||
: augmentedInput;
|
||||
// 组装最终 prompt:系统提示词 → 多轮历史 → RAG 增强后的当前提问
|
||||
StringBuilder promptBuilder = new StringBuilder();
|
||||
if (agentVo != null && StringUtils.isNotBlank(agentVo.getSystemPrompt())) {
|
||||
promptBuilder.append(agentVo.getSystemPrompt()).append("\n\n");
|
||||
}
|
||||
String historyText = formatHistoryMessages(chatRequest.getContextMessages(), chatRequest.getContent());
|
||||
if (StringUtils.isNotBlank(historyText)) {
|
||||
promptBuilder.append("以下是本次会话的历史对话,请结合上下文理解用户最新提问:\n")
|
||||
.append(historyText).append("\n\n");
|
||||
}
|
||||
promptBuilder.append(augmentedInput);
|
||||
String prompt = promptBuilder.toString();
|
||||
|
||||
String tokenValue = chatRequest.getTokenValue();
|
||||
|
||||
@@ -316,8 +327,9 @@ public class ChatServiceFacade implements IChatService {
|
||||
* 兜底 MCP 工具装配(无智能体时使用,保留原有 3 个硬编码客户端逻辑)
|
||||
*/
|
||||
private ToolProvider buildDefaultMcpToolProvider(Long userId) {
|
||||
String npxCommand = resolveNpxCommand();
|
||||
McpTransport playwrightTransport = new StdioMcpTransport.Builder()
|
||||
.command(List.of("C:\\Program Files\\nodejs\\npx.cmd", "-y", "@playwright/mcp@latest"))
|
||||
.command(List.of(npxCommand, "-y", "@playwright/mcp@latest"))
|
||||
.logEvents(true)
|
||||
.build();
|
||||
McpClient playwrightMcpClient = new DefaultMcpClient.Builder()
|
||||
@@ -327,7 +339,7 @@ public class ChatServiceFacade implements IChatService {
|
||||
|
||||
String userDir = System.getProperty("user.dir");
|
||||
McpTransport filesystemTransport = new StdioMcpTransport.Builder()
|
||||
.command(List.of("C:\\Program Files\\nodejs\\npx.cmd", "-y",
|
||||
.command(List.of(npxCommand, "-y",
|
||||
"@modelcontextprotocol/server-filesystem", userDir))
|
||||
.logEvents(true)
|
||||
.build();
|
||||
@@ -341,6 +353,14 @@ public class ChatServiceFacade implements IChatService {
|
||||
.build();
|
||||
}
|
||||
|
||||
private String resolveNpxCommand() {
|
||||
String configured = System.getProperty("mcp.npx.command");
|
||||
if (StringUtils.isNotBlank(configured)) return configured;
|
||||
String fromEnv = System.getenv("MCP_NPX_COMMAND");
|
||||
if (StringUtils.isNotBlank(fromEnv)) return fromEnv;
|
||||
return System.getProperty("os.name", "").toLowerCase().contains("win") ? "npx.cmd" : "npx";
|
||||
}
|
||||
|
||||
/**
|
||||
* 装配磁盘 ShellSkills:智能体勾选了技能名时按名过滤,否则加载全部。
|
||||
* 无 skills 时返回 null(调用方据此跳过 SkillsAgent 的 toolProvider 注入)
|
||||
@@ -476,27 +496,7 @@ public class ChatServiceFacade implements IChatService {
|
||||
messages.add(SystemMessage.from(agentVo.getSystemPrompt()));
|
||||
}
|
||||
|
||||
// 1. 初始化当前用户消息
|
||||
UserMessage userMessage = UserMessage.userMessage(chatRequest.getContent());
|
||||
|
||||
// 2. 知识库检索增强 (RAG):智能体的 knowledgeIds 优先,回退到请求的 knowledgeId
|
||||
List<Long> knowledgeIds = collectKnowledgeIds(chatRequest, agentVo);
|
||||
if (knowledgeIds != null && !knowledgeIds.isEmpty()) {
|
||||
RetrievalAugmentor augmentor = buildMultiKnowledgeAugmentor(knowledgeIds);
|
||||
if (augmentor != null) {
|
||||
log.info("执行多知识库 RAG 流程: kids={}", knowledgeIds);
|
||||
Metadata metadata = Metadata.from(userMessage, chatRequest.getSessionId(), new ArrayList<>());
|
||||
AugmentationRequest augmentationRequest = new AugmentationRequest(userMessage, metadata);
|
||||
AugmentationResult result = augmentor.augment(augmentationRequest);
|
||||
ChatMessage augmented = result.chatMessage();
|
||||
if (augmented instanceof UserMessage) {
|
||||
userMessage = (UserMessage) augmented;
|
||||
log.debug("RAG 增强完成,UserMessage 已注入背景知识");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 从数据库查询历史对话消息(放在前面)
|
||||
// 1. 从数据库查询历史对话消息(放在前面)
|
||||
if (chatRequest.getSessionId() != null) {
|
||||
MessageWindowChatMemory memory = createChatMemory(chatRequest.getSessionId());
|
||||
if (memory != null) {
|
||||
@@ -508,12 +508,37 @@ public class ChatServiceFacade implements IChatService {
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 添加经过增强的用户消息(放在最后)
|
||||
messages.add(userMessage);
|
||||
// 2. 添加当前用户消息(放在最后;RAG 增强在 handleAgentChat 中统一执行,避免重复检索)
|
||||
messages.add(UserMessage.userMessage(chatRequest.getContent()));
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将上下文消息格式化为多轮对话文本(供只接受 String 输入的 Supervisor 使用)。
|
||||
* 跳过 SystemMessage(系统提示词单独前置)与最后一条当前用户消息(单独做 RAG 增强后拼接)。
|
||||
*/
|
||||
private String formatHistoryMessages(List<ChatMessage> contextMessages, String currentContent) {
|
||||
if (contextMessages == null || contextMessages.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int limit = contextMessages.size();
|
||||
// 最后一条是当前用户消息,不纳入历史(避免与增强后的输入重复)
|
||||
if (limit > 0 && contextMessages.get(limit - 1) instanceof UserMessage) {
|
||||
limit--;
|
||||
}
|
||||
for (int i = 0; i < limit; i++) {
|
||||
ChatMessage msg = contextMessages.get(i);
|
||||
if (msg instanceof UserMessage userMsg) {
|
||||
sb.append("用户: ").append(userMsg.singleText()).append("\n");
|
||||
} else if (msg instanceof AiMessage aiMsg) {
|
||||
sb.append("助手: ").append(aiMsg.text()).append("\n");
|
||||
}
|
||||
}
|
||||
return sb.toString().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 汇总本次对话要检索的知识库ID列表:智能体绑定的 knowledgeIds 优先,回退到请求的 knowledgeId
|
||||
*/
|
||||
@@ -580,18 +605,35 @@ public class ChatServiceFacade implements IChatService {
|
||||
|
||||
@Override
|
||||
public List<Content> retrieve(Query query) {
|
||||
List<Content> all = new ArrayList<>();
|
||||
for (ContentRetriever r : delegates) {
|
||||
List<CompletableFuture<List<Content>>> futures = delegates.stream()
|
||||
.map(r -> CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
List<Content> part = r.retrieve(query);
|
||||
if (part != null) {
|
||||
all.addAll(part);
|
||||
}
|
||||
return part == null ? List.<Content>of() : part;
|
||||
} catch (Exception e) {
|
||||
log.warn("复合检索子检索器异常: {}", e.getMessage());
|
||||
return List.<Content>of();
|
||||
}
|
||||
})).toList();
|
||||
Map<String, Content> unique = new LinkedHashMap<>();
|
||||
for (CompletableFuture<List<Content>> future : futures) {
|
||||
for (Content content : future.join()) {
|
||||
String key = content.textSegment().metadata().getString("kid") + "|"
|
||||
+ content.textSegment().metadata().getString("docId") + "|"
|
||||
+ content.textSegment().metadata().getString("fid");
|
||||
if (key.endsWith("null|null|null")) key = content.textSegment().text();
|
||||
unique.putIfAbsent(key, content);
|
||||
}
|
||||
}
|
||||
return all;
|
||||
List<Content> bounded = new ArrayList<>();
|
||||
int chars = 0;
|
||||
for (Content content : unique.values()) {
|
||||
int next = content.textSegment().text().length();
|
||||
if (bounded.size() >= 20 || chars + next > 24000) break;
|
||||
bounded.add(content);
|
||||
chars += next;
|
||||
}
|
||||
return bounded;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -647,6 +689,17 @@ public class ChatServiceFacade implements IChatService {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onPartialThinking(PartialThinking partialThinking) {
|
||||
// 发送推理内容到 SSE(前端通过 reasoning 事件监听)
|
||||
SseMessageUtils.sendReasoning(userId, partialThinking.text());
|
||||
|
||||
// 转发给外部 handler
|
||||
if (externalHandler != null) {
|
||||
externalHandler.onPartialThinking(partialThinking);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCompleteResponse(ChatResponse completeResponse) {
|
||||
try {
|
||||
|
||||
@@ -32,10 +32,13 @@ public class OllamaServiceImpl implements AbstractChatService {
|
||||
|
||||
@Override
|
||||
public StreamingChatModel buildStreamingChatModel(ChatModelVo chatModelVo, ChatRequest chatRequest) {
|
||||
boolean thinkingEnabled = Boolean.TRUE.equals(chatRequest.getEnableThinking());
|
||||
return OllamaStreamingChatModel.builder()
|
||||
.baseUrl(chatModelVo.getApiHost())
|
||||
.modelName(chatModelVo.getModelName())
|
||||
.listeners(List.of(new MyChatModelListener()))
|
||||
.think(thinkingEnabled)
|
||||
.returnThinking(thinkingEnabled)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import java.util.Set;
|
||||
* @Description: 阿里百炼基础嵌入模型(兼容openai)
|
||||
*/
|
||||
@Component("alibailian")
|
||||
@org.springframework.context.annotation.Scope("prototype")
|
||||
public class AliBaiLianBaseEmbedProvider extends OpenAiEmbeddingProvider {
|
||||
|
||||
private ChatModelVo chatModelVo;
|
||||
|
||||
@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
|
||||
* 实现了MultiModalEmbedModelService接口,提供文本、图像和视频的嵌入向量生成服务
|
||||
*/
|
||||
@Component("bailianMultiModel")
|
||||
@org.springframework.context.annotation.Scope("prototype")
|
||||
@Slf4j
|
||||
public class AliBaiLianMultiEmbeddingProvider implements MultiModalEmbedModelService {
|
||||
private final OkHttpClient okHttpClient;
|
||||
|
||||
@@ -12,6 +12,7 @@ import org.springframework.stereotype.Component;
|
||||
* @date 2026/3/21
|
||||
*/
|
||||
@Component("minimax")
|
||||
@org.springframework.context.annotation.Scope("prototype")
|
||||
public class MinimaxEmbeddingProvider extends OpenAiEmbeddingProvider {
|
||||
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import java.util.Set;
|
||||
* @Description: Ollama嵌入模型
|
||||
*/
|
||||
@Component("ollama")
|
||||
@org.springframework.context.annotation.Scope("prototype")
|
||||
public class OllamaEmbeddingProvider implements BaseEmbedModelService {
|
||||
private ChatModelVo chatModelVo;
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import java.util.Set;
|
||||
* @Description: OpenAi嵌入模型
|
||||
*/
|
||||
@Component("openai")
|
||||
@org.springframework.context.annotation.Scope("prototype")
|
||||
public class OpenAiEmbeddingProvider implements BaseEmbedModelService {
|
||||
protected ChatModelVo chatModelVo;
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import org.springframework.stereotype.Component;
|
||||
* @Description: 硅基流动(兼容 OpenAi)
|
||||
*/
|
||||
@Component("siliconflow")
|
||||
@org.springframework.context.annotation.Scope("prototype")
|
||||
public class SiliconFlowEmbeddingProvider extends OpenAiEmbeddingProvider {
|
||||
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import java.util.Set;
|
||||
* @Description: 智谱AI嵌入模型
|
||||
*/
|
||||
@Component("zhipu")
|
||||
@org.springframework.context.annotation.Scope("prototype")
|
||||
public class ZhipuAiEmbeddingProvider implements BaseEmbedModelService {
|
||||
protected ChatModelVo chatModelVo;
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package org.ruoyi.service.knowledge;
|
||||
|
||||
import org.ruoyi.common.core.exception.ServiceException;
|
||||
|
||||
/** Immutable snapshot of the split settings used for one parse operation. */
|
||||
public record DocumentSplitConfig(String separator, int blockSize, int overlap, String fileType) {
|
||||
|
||||
public static final int DEFAULT_BLOCK_SIZE = 1000;
|
||||
public static final int DEFAULT_OVERLAP = 50;
|
||||
|
||||
public DocumentSplitConfig {
|
||||
if (blockSize <= 0) {
|
||||
throw new ServiceException("文本块大小必须大于0");
|
||||
}
|
||||
if (overlap < 0 || overlap >= blockSize) {
|
||||
throw new ServiceException("重叠字符数必须大于等于0且小于文本块大小");
|
||||
}
|
||||
fileType = fileType == null ? "" : fileType.strip().replaceFirst("^\\.", "").toLowerCase();
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import org.ruoyi.common.mybatis.core.page.PageQuery;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
@@ -79,4 +80,6 @@ public interface IKnowledgeAttachService {
|
||||
* @param id 附件ID
|
||||
*/
|
||||
void parse(Long id);
|
||||
|
||||
KnowledgeReparseVo reparseKnowledge(Long knowledgeId);
|
||||
}
|
||||
|
||||
@@ -10,5 +10,5 @@ public interface ResourceLoader {
|
||||
|
||||
String getContent(InputStream inputStream);
|
||||
|
||||
List<String> getChunkList(String content, String kid);
|
||||
List<String> getChunkList(String content, DocumentSplitConfig config);
|
||||
}
|
||||
|
||||
@@ -11,8 +11,8 @@ public interface TextSplitter {
|
||||
* 文本切分
|
||||
*
|
||||
* @param content 文本内容
|
||||
* @param kid 知识库id
|
||||
* @param config 本次解析的分片配置快照
|
||||
* @return 切分后的文本列表
|
||||
*/
|
||||
List<String> split(String content, String kid);
|
||||
List<String> split(String content, DocumentSplitConfig config);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package org.ruoyi.service.knowledge.impl;
|
||||
|
||||
import cn.hutool.core.collection.CollUtil;
|
||||
import cn.hutool.core.util.RandomUtil;
|
||||
import cn.hutool.crypto.digest.DigestUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
@@ -11,6 +12,7 @@ import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.common.chat.service.chat.IChatModelService;
|
||||
import org.ruoyi.enums.KnowledgeAttachStatus;
|
||||
import org.ruoyi.common.core.domain.dto.OssDTO;
|
||||
import org.ruoyi.common.core.exception.ServiceException;
|
||||
import org.ruoyi.common.core.service.OssService;
|
||||
import org.ruoyi.common.core.utils.MapstructUtils;
|
||||
import org.ruoyi.common.core.utils.SpringUtils;
|
||||
@@ -25,13 +27,16 @@ import org.ruoyi.domain.entity.knowledge.KnowledgeFragment;
|
||||
import org.ruoyi.domain.vo.knowledge.DocFragmentCountVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
|
||||
import org.ruoyi.factory.ResourceLoaderFactory;
|
||||
import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper;
|
||||
import org.ruoyi.mapper.knowledge.KnowledgeFragmentMapper;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeAttachService;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
|
||||
import org.ruoyi.service.knowledge.ResourceLoader;
|
||||
import org.ruoyi.service.knowledge.DocumentSplitConfig;
|
||||
import org.ruoyi.service.vector.VectorStoreService;
|
||||
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
@@ -60,6 +65,7 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
private final ResourceLoaderFactory resourceLoaderFactory;
|
||||
private final VectorStoreService vectorStoreService;
|
||||
private final OssService ossService;
|
||||
private final KnowledgeRetrievalService knowledgeRetrievalService;
|
||||
|
||||
@Override
|
||||
public KnowledgeAttachVo queryById(Long id) {
|
||||
@@ -126,18 +132,44 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
|
||||
@Override
|
||||
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
|
||||
// 删除附件前,同步清理其片段记录与向量库中的向量
|
||||
List<KnowledgeAttach> attaches = baseMapper.selectByIds(ids);
|
||||
for (KnowledgeAttach attach : attaches) {
|
||||
String docId = attach.getDocId();
|
||||
String kid = String.valueOf(attach.getKnowledgeId());
|
||||
vectorStoreService.removeByDocId(docId, kid);
|
||||
knowledgeFragmentMapper.delete(
|
||||
Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
|
||||
if (attach.getOssId() != null) {
|
||||
ossService.deleteFile(attach.getOssId());
|
||||
}
|
||||
knowledgeRetrievalService.invalidateKnowledge(kid);
|
||||
}
|
||||
return baseMapper.deleteByIds(ids) > 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void upload(KnowledgeInfoUploadBo bo) {
|
||||
MultipartFile file = bo.getFile();
|
||||
final String fileHash;
|
||||
try (InputStream input = file.getInputStream()) {
|
||||
fileHash = DigestUtil.sha256Hex(input);
|
||||
} catch (Exception e) {
|
||||
throw new ServiceException("计算文件摘要失败", e);
|
||||
}
|
||||
boolean duplicate = baseMapper.exists(Wrappers.<KnowledgeAttach>lambdaQuery()
|
||||
.eq(KnowledgeAttach::getKnowledgeId, bo.getKnowledgeId())
|
||||
.eq(KnowledgeAttach::getFileHash, fileHash));
|
||||
if (duplicate) {
|
||||
throw new ServiceException("该文件已上传,请勿重复提交");
|
||||
}
|
||||
OssDTO ossDTO = ossService.uploadFile(file);
|
||||
|
||||
KnowledgeAttach knowledgeAttach = new KnowledgeAttach();
|
||||
knowledgeAttach.setKnowledgeId(bo.getKnowledgeId());
|
||||
knowledgeAttach.setOssId(ossDTO.getOssId());
|
||||
knowledgeAttach.setDocId(RandomUtil.randomString(10));
|
||||
knowledgeAttach.setFileHash(fileHash);
|
||||
knowledgeAttach.setName(ossDTO.getOriginalName());
|
||||
knowledgeAttach.setType(ossDTO.getFileSuffix());
|
||||
knowledgeAttach.setStatus(KnowledgeAttachStatus.WAITING.getCode()); // 待解析
|
||||
@@ -154,10 +186,17 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
@Override
|
||||
public void parse(Long id) {
|
||||
KnowledgeAttach attach = baseMapper.selectById(id);
|
||||
if (attach == null || (!KnowledgeAttachStatus.WAITING.getCode().equals(attach.getStatus()) && !KnowledgeAttachStatus.FAILED.getCode().equals(attach.getStatus()))) {
|
||||
if (attach == null || KnowledgeAttachStatus.PARSING.getCode().equals(attach.getStatus())) {
|
||||
return;
|
||||
}
|
||||
|
||||
int claimed = baseMapper.update(null, Wrappers.<KnowledgeAttach>lambdaUpdate()
|
||||
.set(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode())
|
||||
.set(KnowledgeAttach::getRemark, null)
|
||||
.eq(KnowledgeAttach::getId, id)
|
||||
.ne(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode()));
|
||||
if (claimed == 0) return;
|
||||
|
||||
try {
|
||||
attach.setStatus(KnowledgeAttachStatus.PARSING.getCode()); // 解析中
|
||||
baseMapper.updateById(attach);
|
||||
@@ -166,6 +205,16 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
|
||||
Long knowledgeId = attach.getKnowledgeId();
|
||||
String docId = attach.getDocId();
|
||||
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
|
||||
if (knowledgeInfoVo == null) {
|
||||
throw new ServiceException("知识库不存在: " + knowledgeId);
|
||||
}
|
||||
int blockSize = knowledgeInfoVo.getTextBlockSize() == null
|
||||
? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : knowledgeInfoVo.getTextBlockSize().intValue();
|
||||
int overlap = knowledgeInfoVo.getOverlapChar() == null
|
||||
? DocumentSplitConfig.DEFAULT_OVERLAP : knowledgeInfoVo.getOverlapChar().intValue();
|
||||
DocumentSplitConfig splitConfig = new DocumentSplitConfig(
|
||||
knowledgeInfoVo.getSeparator(), blockSize, overlap, attach.getType());
|
||||
|
||||
// 获取文件信息并下载
|
||||
List<OssDTO> ossDTOs = ossService.selectByIds(String.valueOf(attach.getOssId()));
|
||||
@@ -178,28 +227,27 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
try (InputStream inputStream = new URL(ossDTO.getUrl()).openStream()) {
|
||||
content = resourceLoader.getContent(inputStream);
|
||||
}
|
||||
List<String> chunkList = resourceLoader.getChunkList(content, String.valueOf(knowledgeId));
|
||||
List<String> chunkList = resourceLoader.getChunkList(content, splitConfig);
|
||||
|
||||
if (CollUtil.isEmpty(chunkList)) {
|
||||
throw new RuntimeException("文档分片结果为空,请检查文档内容或分片器是否支持该文件类型");
|
||||
}
|
||||
|
||||
// 重新解析前先清理旧的向量数据,避免向量重复累积
|
||||
List<String> fids = new ArrayList<>();
|
||||
List<KnowledgeFragment> knowledgeFragmentList = new ArrayList<>();
|
||||
if (CollUtil.isNotEmpty(chunkList)) {
|
||||
for (int i = 0; i < chunkList.size(); i++) {
|
||||
String fid = RandomUtil.randomString(10);
|
||||
fids.add(fid);
|
||||
KnowledgeFragment knowledgeFragment = new KnowledgeFragment();
|
||||
knowledgeFragment.setKnowledgeId(knowledgeId);
|
||||
knowledgeFragment.setDocId(docId);
|
||||
knowledgeFragment.setFid(fid);
|
||||
knowledgeFragment.setIdx(i);
|
||||
knowledgeFragment.setContent(chunkList.get(i));
|
||||
knowledgeFragment.setCreateTime(new Date());
|
||||
knowledgeFragmentList.add(knowledgeFragment);
|
||||
}
|
||||
knowledgeFragmentMapper.delete(Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
|
||||
knowledgeFragmentMapper.insertBatch(knowledgeFragmentList);
|
||||
log.info("文档切片并入库完成,共计 {} 个片段。id: {}", chunkList.size(), id);
|
||||
}
|
||||
|
||||
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
|
||||
ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel());
|
||||
|
||||
StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo();
|
||||
@@ -211,7 +259,26 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
storeEmbeddingBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModel());
|
||||
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
|
||||
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
|
||||
try {
|
||||
// 写入新向量前,先按 docId 清理该文档的旧向量:
|
||||
// 历史数据的片段 fid 为迁移脚本回填的 MD5 值,与向量库中实际存储的 fid 不一致,
|
||||
// 按 fid 删除无法命中旧向量,会导致重复向量累积;按 docId 清理对三种向量库均一致有效。
|
||||
vectorStoreService.removeByDocId(docId, String.valueOf(knowledgeId));
|
||||
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
|
||||
} catch (Exception vectorError) {
|
||||
for (String newFid : fids) {
|
||||
try {
|
||||
vectorStoreService.removeByFid(newFid, String.valueOf(knowledgeId));
|
||||
} catch (Exception cleanupError) {
|
||||
log.error("补偿删除新向量失败, kid={}, fid={}", knowledgeId, newFid, cleanupError);
|
||||
}
|
||||
}
|
||||
throw vectorError;
|
||||
}
|
||||
|
||||
knowledgeFragmentMapper.delete(Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
|
||||
knowledgeFragmentMapper.insertBatch(knowledgeFragmentList);
|
||||
knowledgeRetrievalService.invalidateKnowledge(String.valueOf(knowledgeId));
|
||||
|
||||
attach.setStatus(KnowledgeAttachStatus.COMPLETED.getCode()); // 已完成
|
||||
baseMapper.updateById(attach);
|
||||
@@ -223,4 +290,22 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
baseMapper.updateById(attach);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public KnowledgeReparseVo reparseKnowledge(Long knowledgeId) {
|
||||
List<KnowledgeAttach> attachments = baseMapper.selectList(
|
||||
Wrappers.<KnowledgeAttach>lambdaQuery().eq(KnowledgeAttach::getKnowledgeId, knowledgeId));
|
||||
int submitted = 0;
|
||||
int skipped = 0;
|
||||
IKnowledgeAttachService proxy = SpringUtils.getBean(IKnowledgeAttachService.class);
|
||||
for (KnowledgeAttach attachment : attachments) {
|
||||
if (KnowledgeAttachStatus.PARSING.getCode().equals(attachment.getStatus())) {
|
||||
skipped++;
|
||||
} else {
|
||||
proxy.parse(attachment.getId());
|
||||
submitted++;
|
||||
}
|
||||
}
|
||||
return new KnowledgeReparseVo(submitted, skipped, attachments.size());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ public class KnowledgeFragmentServiceImpl implements IKnowledgeFragmentService {
|
||||
private final IKnowledgeInfoService knowledgeInfoService;
|
||||
private final IChatModelService chatModelService;
|
||||
private final KnowledgeRetrievalService knowledgeRetrievalService;
|
||||
private final org.ruoyi.service.vector.VectorStoreService vectorStoreService;
|
||||
|
||||
/**
|
||||
* 查询知识片段
|
||||
@@ -114,7 +115,11 @@ public class KnowledgeFragmentServiceImpl implements IKnowledgeFragmentService {
|
||||
public Boolean updateByBo(KnowledgeFragmentBo bo) {
|
||||
KnowledgeFragment update = MapstructUtils.convert(bo, KnowledgeFragment.class);
|
||||
validEntityBeforeSave(update);
|
||||
return baseMapper.updateById(update) > 0;
|
||||
boolean updated = baseMapper.updateById(update) > 0;
|
||||
if (updated && update.getKnowledgeId() != null) {
|
||||
knowledgeRetrievalService.invalidateKnowledge(String.valueOf(update.getKnowledgeId()));
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -136,6 +141,14 @@ public class KnowledgeFragmentServiceImpl implements IKnowledgeFragmentService {
|
||||
if(isValid){
|
||||
//TODO 做一些业务上的校验,判断是否需要校验
|
||||
}
|
||||
// 删除 DB 片段前,同步删除向量库中对应向量
|
||||
List<KnowledgeFragment> fragments = baseMapper.selectByIds(ids);
|
||||
for (KnowledgeFragment fragment : fragments) {
|
||||
if (StringUtils.isNotBlank(fragment.getFid()) && fragment.getKnowledgeId() != null) {
|
||||
vectorStoreService.removeByFid(fragment.getFid(), String.valueOf(fragment.getKnowledgeId()));
|
||||
knowledgeRetrievalService.invalidateKnowledge(String.valueOf(fragment.getKnowledgeId()));
|
||||
}
|
||||
}
|
||||
return baseMapper.deleteByIds(ids) > 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,10 @@ import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper;
|
||||
import org.ruoyi.mapper.knowledge.KnowledgeInfoMapper;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
|
||||
import org.ruoyi.service.knowledge.DocumentSplitConfig;
|
||||
import org.ruoyi.common.core.service.OssService;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -36,6 +40,12 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
|
||||
|
||||
private final KnowledgeAttachMapper knowledgeAttachMapper;
|
||||
|
||||
private final org.ruoyi.mapper.knowledge.KnowledgeFragmentMapper knowledgeFragmentMapper;
|
||||
|
||||
private final org.ruoyi.service.vector.VectorStoreService vectorStoreService;
|
||||
private final KnowledgeRetrievalService knowledgeRetrievalService;
|
||||
private final OssService ossService;
|
||||
|
||||
/**
|
||||
* 查询知识库
|
||||
*
|
||||
@@ -97,10 +107,14 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
|
||||
*/
|
||||
private void fillDocumentCount(List<KnowledgeInfoVo> records) {
|
||||
if (records == null || records.isEmpty()) return;
|
||||
for (KnowledgeInfoVo vo : records) {
|
||||
int count = knowledgeAttachMapper.countByKnowledgeId(vo.getId());
|
||||
vo.setDocumentCount(count);
|
||||
List<Long> ids = records.stream().map(KnowledgeInfoVo::getId).toList();
|
||||
Map<Long, Integer> counts = new java.util.HashMap<>();
|
||||
for (Map<String, Object> row : knowledgeAttachMapper.countByKnowledgeIds(ids)) {
|
||||
Number kid = (Number) (row.get("knowledgeId") != null ? row.get("knowledgeId") : row.get("knowledgeid"));
|
||||
Number count = (Number) (row.get("documentCount") != null ? row.get("documentCount") : row.get("documentcount"));
|
||||
if (kid != null && count != null) counts.put(kid.longValue(), count.intValue());
|
||||
}
|
||||
records.forEach(vo -> vo.setDocumentCount(counts.getOrDefault(vo.getId(), 0)));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -130,13 +144,20 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
|
||||
public Boolean updateByBo(KnowledgeInfoBo bo) {
|
||||
KnowledgeInfo update = MapstructUtils.convert(bo, KnowledgeInfo.class);
|
||||
validEntityBeforeSave(update);
|
||||
return baseMapper.updateById(update) > 0;
|
||||
boolean updated = baseMapper.updateById(update) > 0;
|
||||
if (updated) knowledgeRetrievalService.invalidateKnowledge(String.valueOf(bo.getId()));
|
||||
return updated;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存前的数据校验
|
||||
*/
|
||||
private void validEntityBeforeSave(KnowledgeInfo entity){
|
||||
int blockSize = entity.getTextBlockSize() == null
|
||||
? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : entity.getTextBlockSize().intValue();
|
||||
int overlap = entity.getOverlapChar() == null
|
||||
? DocumentSplitConfig.DEFAULT_OVERLAP : entity.getOverlapChar().intValue();
|
||||
new DocumentSplitConfig(entity.getSeparator(), blockSize, overlap, "");
|
||||
//TODO 做一些数据校验,如唯一约束
|
||||
}
|
||||
|
||||
@@ -148,10 +169,33 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
|
||||
* @return 是否删除成功
|
||||
*/
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
|
||||
if(isValid){
|
||||
//TODO 做一些业务上的校验,判断是否需要校验
|
||||
}
|
||||
for (Long kid : ids) {
|
||||
KnowledgeInfo info = baseMapper.selectById(kid);
|
||||
// 1. 删除向量库中该知识库的所有向量(按文档逐个清理,三种向量库行为一致)
|
||||
List<org.ruoyi.domain.entity.knowledge.KnowledgeAttach> attaches = knowledgeAttachMapper.selectList(
|
||||
Wrappers.lambdaQuery(org.ruoyi.domain.entity.knowledge.KnowledgeAttach.class)
|
||||
.eq(org.ruoyi.domain.entity.knowledge.KnowledgeAttach::getKnowledgeId, kid));
|
||||
vectorStoreService.removeById(String.valueOf(kid), info == null ? null : info.getVectorModel());
|
||||
List<Long> ossIds = attaches.stream()
|
||||
.map(org.ruoyi.domain.entity.knowledge.KnowledgeAttach::getOssId)
|
||||
.filter(java.util.Objects::nonNull).toList();
|
||||
if (!ossIds.isEmpty()) {
|
||||
for (Long ossId : ossIds) {
|
||||
ossService.deleteFile(ossId);
|
||||
}
|
||||
}
|
||||
// 2. 删除该知识库下的附件与片段记录
|
||||
knowledgeAttachMapper.delete(Wrappers.lambdaQuery(org.ruoyi.domain.entity.knowledge.KnowledgeAttach.class)
|
||||
.eq(org.ruoyi.domain.entity.knowledge.KnowledgeAttach::getKnowledgeId, kid));
|
||||
knowledgeFragmentMapper.delete(Wrappers.lambdaQuery(org.ruoyi.domain.entity.knowledge.KnowledgeFragment.class)
|
||||
.eq(org.ruoyi.domain.entity.knowledge.KnowledgeFragment::getKnowledgeId, kid));
|
||||
knowledgeRetrievalService.invalidateKnowledge(String.valueOf(kid));
|
||||
}
|
||||
return baseMapper.deleteByIds(ids) > 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,13 +3,16 @@ package org.ruoyi.service.knowledge.impl.loader;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.service.knowledge.ResourceLoader;
|
||||
import org.ruoyi.service.knowledge.DocumentSplitConfig;
|
||||
import org.ruoyi.service.knowledge.TextSplitter;
|
||||
import org.ruoyi.common.core.exception.ServiceException;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
@@ -21,20 +24,20 @@ public class CodeFileLoader implements ResourceLoader {
|
||||
@Override
|
||||
public String getContent(InputStream inputStream) {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
try (InputStreamReader reader = new InputStreamReader(inputStream);
|
||||
try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
|
||||
BufferedReader bufferedReader = new BufferedReader(reader)) {
|
||||
String line;
|
||||
while ((line = bufferedReader.readLine()) != null) {
|
||||
stringBuffer.append(line).append("\n");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
throw new ServiceException("读取代码文件失败", e);
|
||||
}
|
||||
return stringBuffer.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getChunkList(String content, String kid) {
|
||||
return textSplitter.split(content, kid);
|
||||
public List<String> getChunkList(String content, DocumentSplitConfig config) {
|
||||
return textSplitter.split(content, config);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package org.ruoyi.service.knowledge.impl.loader;
|
||||
|
||||
import org.ruoyi.service.knowledge.ResourceLoader;
|
||||
import org.ruoyi.service.knowledge.DocumentSplitConfig;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
@@ -12,7 +13,7 @@ public class CsvFileLoader implements ResourceLoader {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getChunkList(String content, String kid) {
|
||||
public List<String> getChunkList(String content, DocumentSplitConfig config) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import dev.langchain4j.data.document.parser.apache.tika.ApacheTikaDocumentParser
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.service.knowledge.ResourceLoader;
|
||||
import org.ruoyi.service.knowledge.DocumentSplitConfig;
|
||||
import org.ruoyi.service.knowledge.TextSplitter;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -38,7 +39,7 @@ public class ExcelFileLoader implements ResourceLoader {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getChunkList(String content, String kid) {
|
||||
return textSplitter.split(content, kid);
|
||||
public List<String> getChunkList(String content, DocumentSplitConfig config) {
|
||||
return textSplitter.split(content, config);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package org.ruoyi.service.knowledge.impl.loader;
|
||||
|
||||
import org.ruoyi.service.knowledge.ResourceLoader;
|
||||
import org.ruoyi.service.knowledge.DocumentSplitConfig;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
@@ -12,7 +13,7 @@ public class FolderLoader implements ResourceLoader {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getChunkList(String content, String kid) {
|
||||
public List<String> getChunkList(String content, DocumentSplitConfig config) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user