mirror of
https://gitcode.com/ageerle/ruoyi-ai.git
synced 2026-04-24 09:13:39 +00:00
Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9a7b727413 | ||
|
|
b8d16b7669 | ||
|
|
058a4aee2a | ||
|
|
1b50c7f9f1 | ||
|
|
e7f53fd55f | ||
|
|
07bdc5e585 | ||
|
|
e1b8a5f011 | ||
|
|
80ca76ea37 | ||
|
|
ccbf5c9520 | ||
|
|
1208c46cca | ||
|
|
06a63c377e | ||
|
|
0fa25032a3 | ||
|
|
28ad29d6ed |
46
docs/script/sql/update/updat-0420.sql
Normal file
46
docs/script/sql/update/updat-0420.sql
Normal file
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
Navicat Premium Dump SQL
|
||||
|
||||
Source Server : localhost-mysql
|
||||
Source Server Type : MySQL
|
||||
Source Server Version : 80045 (8.0.45)
|
||||
Source Host : localhost:3306
|
||||
Source Schema : ruoyi-ai
|
||||
|
||||
Target Server Type : MySQL
|
||||
Target Server Version : 80045 (8.0.45)
|
||||
File Encoding : 65001
|
||||
|
||||
Date: 20/04/2026 15:30:00
|
||||
*/
|
||||
|
||||
SET NAMES utf8mb4;
|
||||
SET FOREIGN_KEY_CHECKS = 0;
|
||||
|
||||
-- ----------------------------
|
||||
-- 新增:重排序模型(chat_model)
|
||||
-- ----------------------------
|
||||
INSERT INTO `chat_model`
|
||||
(id, category, model_name, provider_code, model_describe, model_dimension, model_show, api_host, api_key, create_dept, create_by, create_time, update_by, update_time, remark, tenant_id)
|
||||
VALUES(2045071617578237953, 'rerank', 'rerank', 'zhipu', '智谱重排序', NULL, 'Y', 'https://open.bigmodel.cn', 'e9xx', 103, 1, '2026-04-17 17:27:24', 1, '2026-04-20 15:21:48', '智谱重排序', 0);
|
||||
|
||||
INSERT INTO `chat_model`
|
||||
(id, category, model_name, provider_code, model_describe, model_dimension, model_show, api_host, api_key, create_dept, create_by, create_time, update_by, update_time, remark, tenant_id)
|
||||
VALUES(2046119803482902530, 'rerank', 'qwen3-rerank', 'qianwen', '千问3重排序', NULL, NULL, 'https://dashscope.aliyuncs.com', 'sk-xx', 103, 1, '2026-04-20 14:52:31', 1, '2026-04-20 15:03:13', '千问3文本重排序', 0);
|
||||
|
||||
-- ----------------------------
|
||||
-- 新增:字典类型 - 重排序模型分类
|
||||
-- ----------------------------
|
||||
INSERT INTO `sys_dict_data`
|
||||
(dict_code, tenant_id, dict_sort, dict_label, dict_value, dict_type, css_class, list_class, is_default, create_dept, create_by, create_time, update_by, update_time, remark)
|
||||
VALUES(2045070879435259905, '000000', 4, '重排序', 'rerank', 'chat_model_category', NULL, '#000000', 'N', 103, 1, '2026-04-17 17:24:28', 1, '2026-04-19 01:02:20', '重排序模型');
|
||||
|
||||
-- ----------------------------
|
||||
-- 修改表:knowledge_info 增加重排序相关字段
|
||||
-- ----------------------------
|
||||
ALTER TABLE `knowledge_info` ADD COLUMN `enable_rerank` tinyint DEFAULT 0 NULL COMMENT '是否启用重排序(0否 1是)';
|
||||
ALTER TABLE `knowledge_info` ADD COLUMN `rerank_score_threshold` double NULL COMMENT '重排序相关性分数阈值';
|
||||
ALTER TABLE `knowledge_info` ADD COLUMN `rerank_top_n` int NULL COMMENT '重排序后返回的文档数量';
|
||||
ALTER TABLE `knowledge_info` ADD COLUMN `rerank_model` varchar(100) NULL COMMENT '重排序模型名称';
|
||||
|
||||
SET FOREIGN_KEY_CHECKS = 1;
|
||||
14
docs/script/sql/update/updat-0423.sql
Normal file
14
docs/script/sql/update/updat-0423.sql
Normal file
@@ -0,0 +1,14 @@
|
||||
-- 为知识库信息表新增检索配置字段 (剔除了已存在的重排字段)
|
||||
ALTER TABLE knowledge_info
|
||||
ADD COLUMN similarity_threshold DOUBLE DEFAULT 0.5 COMMENT '相似度阈值'
|
||||
AFTER retrieve_limit;
|
||||
|
||||
ALTER TABLE knowledge_info ADD COLUMN enable_hybrid tinyint(1) DEFAULT 0 COMMENT '是否启用混合检索';
|
||||
ALTER TABLE knowledge_info ADD COLUMN hybrid_alpha double DEFAULT 0.5 COMMENT '混合检索权重比例 (0.0=纯向量, 1.0=纯关键词)';
|
||||
|
||||
-- 为知识片段表增加全文索引及关联ID
|
||||
ALTER TABLE knowledge_fragment ADD COLUMN knowledge_id bigint COMMENT '知识库ID';
|
||||
ALTER TABLE knowledge_fragment ADD FULLTEXT INDEX ft_content (content) WITH PARSER ngram;
|
||||
|
||||
-- 为知识库附件表增加解析状态字段
|
||||
ALTER TABLE `knowledge_attach` ADD COLUMN `status` TINYINT DEFAULT 0 COMMENT '解析状态: 0待解析, 1解析中, 2已解析, 3解析失败';
|
||||
2
pom.xml
2
pom.xml
@@ -58,7 +58,7 @@
|
||||
<langchain4j.community.version>1.13.0-beta23</langchain4j.community.version>
|
||||
<langgraph4j.version>1.5.3</langgraph4j.version>
|
||||
<weaviate.version>1.19.6</weaviate.version>
|
||||
<dify.version>1.2.6</dify.version>
|
||||
<dify.version>1.0.7</dify.version>
|
||||
<!-- gRPC 版本 - 解决 Milvus SDK 依赖冲突 -->
|
||||
<grpc.version>1.62.2</grpc.version>
|
||||
<!-- Apache Commons Compress - 用于POI处理ZIP格式 -->
|
||||
|
||||
@@ -10,6 +10,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.core.task.VirtualThreadTaskExecutor;
|
||||
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
import java.util.concurrent.*;
|
||||
|
||||
/**
|
||||
@@ -22,6 +23,12 @@ import java.util.concurrent.*;
|
||||
@EnableConfigurationProperties(ThreadPoolProperties.class)
|
||||
public class ThreadPoolConfig {
|
||||
|
||||
private final ThreadPoolProperties properties;
|
||||
|
||||
public ThreadPoolConfig(ThreadPoolProperties properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 核心线程数 = cpu 核心数 + 1
|
||||
*/
|
||||
@@ -54,6 +61,22 @@ public class ThreadPoolConfig {
|
||||
return scheduledThreadPoolExecutor;
|
||||
}
|
||||
|
||||
/**
|
||||
* 知识库解析专用异步线程池
|
||||
*/
|
||||
@Bean(name = "knowledgeParseExecutor")
|
||||
public ThreadPoolTaskExecutor knowledgeParseExecutor() {
|
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||
executor.setCorePoolSize(core);
|
||||
executor.setMaxPoolSize(core * 2);
|
||||
executor.setQueueCapacity(properties.getQueueCapacity());
|
||||
executor.setKeepAliveSeconds(properties.getKeepAliveSeconds());
|
||||
executor.setThreadNamePrefix("knowledge-parse-pool-");
|
||||
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
executor.initialize();
|
||||
return executor;
|
||||
}
|
||||
|
||||
/**
|
||||
* 销毁事件
|
||||
* 停止线程池
|
||||
|
||||
@@ -110,6 +110,17 @@ public class KnowledgeAttachController extends BaseController {
|
||||
@PostMapping(value = "/upload")
|
||||
public R<String> upload(KnowledgeInfoUploadBo bo){
|
||||
knowledgeAttachService.upload(bo);
|
||||
return R.ok("上传知识库附件成功!");
|
||||
return R.ok("上传成功!");
|
||||
}
|
||||
|
||||
/**
|
||||
* 手动解析附件内容
|
||||
*
|
||||
* @param id 附件ID
|
||||
*/
|
||||
@PostMapping("/parse/{id}")
|
||||
public R<Void> parse(@PathVariable Long id) {
|
||||
knowledgeAttachService.parse(id);
|
||||
return R.ok();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import jakarta.validation.constraints.*;
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeFragmentBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeFragmentVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeFragmentService;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
@@ -102,4 +103,12 @@ public class KnowledgeFragmentController extends BaseController {
|
||||
@PathVariable Long[] ids) {
|
||||
return toAjax(knowledgeFragmentService.deleteWithValidByIds(List.of(ids), true));
|
||||
}
|
||||
|
||||
/**
|
||||
* 检索测试
|
||||
*/
|
||||
@PostMapping("/retrieval")
|
||||
public R<List<KnowledgeRetrievalVo>> retrieval(@RequestBody KnowledgeFragmentBo bo) {
|
||||
return R.ok(knowledgeFragmentService.retrieval(bo));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,5 +49,44 @@ public class KnowledgeFragmentBo extends BaseEntity {
|
||||
*/
|
||||
private String remark;
|
||||
|
||||
/**
|
||||
* 知识库ID
|
||||
*/
|
||||
private Long knowledgeId;
|
||||
|
||||
/**
|
||||
* 检索内容
|
||||
*/
|
||||
private String query;
|
||||
|
||||
/**
|
||||
* 返回条数
|
||||
*/
|
||||
private Integer topK;
|
||||
|
||||
/**
|
||||
* 相似度阈值
|
||||
*/
|
||||
private Double threshold;
|
||||
|
||||
/**
|
||||
* 是否启用重排
|
||||
*/
|
||||
private Boolean enableRerank;
|
||||
|
||||
/**
|
||||
* 重排模型名称
|
||||
*/
|
||||
private String rerankModel;
|
||||
|
||||
/**
|
||||
* 是否启用混合检索
|
||||
*/
|
||||
private Boolean enableHybrid;
|
||||
|
||||
/**
|
||||
* 混合检索权重 (0.0-1.0)
|
||||
*/
|
||||
private Double hybridAlpha;
|
||||
|
||||
}
|
||||
|
||||
@@ -62,6 +62,11 @@ public class KnowledgeInfoBo extends BaseEntity {
|
||||
*/
|
||||
private Long retrieveLimit;
|
||||
|
||||
/**
|
||||
* 相似度阈值
|
||||
*/
|
||||
private Double similarityThreshold;
|
||||
|
||||
/**
|
||||
* 文本块大小
|
||||
*/
|
||||
@@ -77,10 +82,40 @@ public class KnowledgeInfoBo extends BaseEntity {
|
||||
*/
|
||||
private String embeddingModel;
|
||||
|
||||
/**
|
||||
* 是否启用重排序(0 否 1是)
|
||||
*/
|
||||
private Integer enableRerank;
|
||||
|
||||
/**
|
||||
* 重排序模型名称
|
||||
*/
|
||||
private String rerankModel;
|
||||
|
||||
/**
|
||||
* 重排序后返回的文档数量
|
||||
*/
|
||||
private Integer rerankTopN;
|
||||
|
||||
/**
|
||||
* 重排序相关性分数阈值
|
||||
*/
|
||||
private Double rerankScoreThreshold;
|
||||
|
||||
|
||||
/**
|
||||
* 是否启用混合检索(0 否 1是)
|
||||
*/
|
||||
private Integer enableHybrid;
|
||||
|
||||
/**
|
||||
* 混合检索权重 (0.0-1.0)
|
||||
*/
|
||||
private Double hybridAlpha;
|
||||
|
||||
/**
|
||||
* 备注
|
||||
*/
|
||||
private String remark;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -16,6 +16,11 @@ public class KnowledgeInfoUploadBo {
|
||||
|
||||
private MultipartFile file;
|
||||
|
||||
/**
|
||||
* 是否自动解析 (true: 立即解析, false: 仅上传)
|
||||
*/
|
||||
private Boolean autoParse;
|
||||
|
||||
/**
|
||||
* 生效时间, 为空则立即生效
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package org.ruoyi.domain.bo.rerank;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 重排序请求参数
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-19
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RerankRequest {
|
||||
|
||||
/**
|
||||
* 查询文本
|
||||
*/
|
||||
private String query;
|
||||
|
||||
/**
|
||||
* 候选文档列表
|
||||
*/
|
||||
private List<String> documents;
|
||||
|
||||
/**
|
||||
* 返回的文档数量(topN)
|
||||
* 如果不指定,默认返回所有文档
|
||||
*/
|
||||
private Integer topN;
|
||||
|
||||
/**
|
||||
* 是否返回原始文档内容
|
||||
* 默认为 true
|
||||
*/
|
||||
@Builder.Default
|
||||
private Boolean returnDocuments = true;
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package org.ruoyi.domain.bo.rerank;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 重排序结果
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-19
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RerankResult {
|
||||
|
||||
/**
|
||||
* 重排序后的文档结果列表
|
||||
*/
|
||||
private List<RerankDocument> documents;
|
||||
|
||||
/**
|
||||
* 原始请求中的文档总数
|
||||
*/
|
||||
private Integer totalDocuments;
|
||||
|
||||
/**
|
||||
* 重排序耗时(毫秒)
|
||||
*/
|
||||
private Long durationMs;
|
||||
|
||||
/**
|
||||
* 单个重排序文档结果
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static class RerankDocument {
|
||||
|
||||
/**
|
||||
* 文档在原始列表中的索引位置
|
||||
*/
|
||||
private Integer index;
|
||||
|
||||
/**
|
||||
* 相关性分数(通常 0-1 之间,越高越相关)
|
||||
*/
|
||||
private Double relevanceScore;
|
||||
|
||||
/**
|
||||
* 文档内容
|
||||
*/
|
||||
private String document;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建空结果
|
||||
*/
|
||||
public static RerankResult empty() {
|
||||
return RerankResult.builder()
|
||||
.documents(List.of())
|
||||
.totalDocuments(0)
|
||||
.durationMs(0L)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -51,4 +51,48 @@ public class QueryVectorBo {
|
||||
*/
|
||||
private String baseUrl;
|
||||
|
||||
|
||||
// ========== 重排序相关参数 ==========
|
||||
|
||||
/**
|
||||
* 是否启用重排序
|
||||
* 默认为 false
|
||||
*/
|
||||
private Boolean enableRerank = false;
|
||||
|
||||
/**
|
||||
* 重排序模型名称
|
||||
*/
|
||||
private String rerankModelName;
|
||||
|
||||
/**
|
||||
* 重排序后返回的文档数量(topN)
|
||||
* 如果不指定,默认与 maxResults 相同
|
||||
*/
|
||||
private Integer rerankTopN;
|
||||
|
||||
/**
|
||||
* 重排序相关性分数阈值
|
||||
* 低于此阈值的文档将被过滤
|
||||
*/
|
||||
private Double rerankScoreThreshold;
|
||||
|
||||
// ========== 混合检索与阈值相关参数 ==========
|
||||
|
||||
/**
|
||||
* 相似度阈值 (0.0-1.0)
|
||||
* 应用于向量搜索阶段
|
||||
*/
|
||||
private Double similarityThreshold;
|
||||
|
||||
/**
|
||||
* 是否启用混合检索
|
||||
*/
|
||||
private Boolean enableHybrid = false;
|
||||
|
||||
/**
|
||||
* 混合检索权重 (0.0-1.0)
|
||||
*/
|
||||
private Double hybridAlpha;
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package org.ruoyi.domain.dto.request;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 阿里百炼重排序请求DTO(OpenAI兼容格式)
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-20
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public record AliBaiLianRerankRequest(
|
||||
String model,
|
||||
List<String> documents,
|
||||
String query,
|
||||
@JsonProperty("top_n")
|
||||
Integer topN,
|
||||
String instruct,
|
||||
@JsonProperty("return_documents")
|
||||
Boolean returnDocuments
|
||||
) {
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
/**
|
||||
* 创建文本重排序请求
|
||||
*/
|
||||
public static AliBaiLianRerankRequest create(String modelName, String query,
|
||||
List<String> documents, Integer topN,
|
||||
Boolean returnDocuments) {
|
||||
return new AliBaiLianRerankRequest(
|
||||
modelName,
|
||||
documents,
|
||||
query,
|
||||
topN != null ? topN : documents.size(),
|
||||
null,
|
||||
returnDocuments != null ? returnDocuments : true
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 转换为JSON字符串
|
||||
*/
|
||||
public String toJson() {
|
||||
try {
|
||||
return OBJECT_MAPPER.writeValueAsString(this);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalArgumentException("序列化阿里百炼重排序请求失败", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package org.ruoyi.domain.dto.request;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 智谱AI重排序请求DTO
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-19
|
||||
*/
|
||||
public record ZhipuRerankRequest(
|
||||
String model,
|
||||
String query,
|
||||
List<String> documents,
|
||||
Integer top_n,
|
||||
Boolean return_documents
|
||||
) {
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
/**
|
||||
* 创建智谱重排序请求
|
||||
*/
|
||||
public static ZhipuRerankRequest create(String modelName, String query,
|
||||
List<String> documents, Integer topN,
|
||||
Boolean returnDocuments) {
|
||||
return new ZhipuRerankRequest(
|
||||
modelName,
|
||||
query,
|
||||
documents,
|
||||
topN != null ? topN : documents.size(),
|
||||
returnDocuments != null ? returnDocuments : true
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 转换为JSON字符串
|
||||
*/
|
||||
public String toJson() {
|
||||
try {
|
||||
return OBJECT_MAPPER.writeValueAsString(this);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalArgumentException("序列化智谱重排序请求失败", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package org.ruoyi.domain.dto.response;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.ruoyi.domain.bo.rerank.RerankResult;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 阿里百炼重排序响应DTO(OpenAI兼容格式)
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-20
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record AliBaiLianRerankResponse(
|
||||
String id,
|
||||
String object,
|
||||
List<ResultItem> results,
|
||||
UsageInfo usage
|
||||
) {
|
||||
/**
|
||||
* 单个重排序结果项
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record ResultItem(
|
||||
Integer index,
|
||||
@JsonProperty("relevance_score")
|
||||
Double relevanceScore,
|
||||
Object document
|
||||
) {
|
||||
/**
|
||||
* 获取文档文本内容
|
||||
*/
|
||||
public String getDocumentText() {
|
||||
if (document == null) return null;
|
||||
if (document instanceof String) return (String) document;
|
||||
if (document instanceof Map) {
|
||||
Object text = ((Map<?, ?>) document).get("text");
|
||||
return text != null ? text.toString() : null;
|
||||
}
|
||||
return document.toString();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Token使用信息
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record UsageInfo(
|
||||
@JsonProperty("total_tokens")
|
||||
Integer totalTokens,
|
||||
@JsonProperty("prompt_tokens")
|
||||
Integer promptTokens
|
||||
) {}
|
||||
|
||||
/**
|
||||
* 转换为通用RerankResult
|
||||
*/
|
||||
public RerankResult toRerankResult(int totalDocs, long durationMs) {
|
||||
if (results == null || results.isEmpty()) {
|
||||
return RerankResult.empty();
|
||||
}
|
||||
|
||||
List<RerankResult.RerankDocument> documents = results.stream()
|
||||
.map(item -> RerankResult.RerankDocument.builder()
|
||||
.index(item.index())
|
||||
.relevanceScore(item.relevanceScore())
|
||||
.document(item.getDocumentText())
|
||||
.build())
|
||||
.collect(Collectors.toList());
|
||||
|
||||
return RerankResult.builder()
|
||||
.documents(documents)
|
||||
.totalDocuments(totalDocs)
|
||||
.durationMs(durationMs)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package org.ruoyi.domain.dto.response;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.ruoyi.domain.bo.rerank.RerankResult;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 智谱AI重排序响应DTO
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-19
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record ZhipuRerankResponse(
|
||||
String model,
|
||||
String object,
|
||||
List<ResultItem> results,
|
||||
UsageInfo usage
|
||||
) {
|
||||
/**
|
||||
* 单个重排序结果项
|
||||
*/
|
||||
public record ResultItem(
|
||||
Integer index,
|
||||
@JsonProperty("relevance_score")
|
||||
Double relevanceScore,
|
||||
String document
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Token使用信息
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record UsageInfo(
|
||||
@JsonProperty("total_tokens")
|
||||
Integer totalTokens,
|
||||
@JsonProperty("input_tokens")
|
||||
Integer inputTokens,
|
||||
@JsonProperty("output_tokens")
|
||||
Integer outputTokens
|
||||
) {}
|
||||
|
||||
/**
|
||||
* 转换为通用RerankResult
|
||||
*/
|
||||
public RerankResult toRerankResult(int totalDocs, long durationMs) {
|
||||
if (results == null || results.isEmpty()) {
|
||||
return RerankResult.empty();
|
||||
}
|
||||
|
||||
List<RerankResult.RerankDocument> documents = results.stream()
|
||||
.map(item -> RerankResult.RerankDocument.builder()
|
||||
.index(item.index())
|
||||
.relevanceScore(item.relevanceScore())
|
||||
.document(item.document())
|
||||
.build())
|
||||
.collect(Collectors.toList());
|
||||
|
||||
return RerankResult.builder()
|
||||
.documents(documents)
|
||||
.totalDocuments(totalDocs)
|
||||
.durationMs(durationMs)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -57,5 +57,10 @@ public class KnowledgeAttach extends BaseEntity {
|
||||
*/
|
||||
private String remark;
|
||||
|
||||
/**
|
||||
* 解析状态: 0待解析, 1解析中, 2已解析, 3解析失败
|
||||
*/
|
||||
private Integer status;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -47,5 +47,10 @@ public class KnowledgeFragment extends BaseEntity {
|
||||
*/
|
||||
private String remark;
|
||||
|
||||
/**
|
||||
* 知识库ID
|
||||
*/
|
||||
private Long knowledgeId;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -63,6 +63,11 @@ public class KnowledgeInfo extends BaseEntity {
|
||||
*/
|
||||
private Long retrieveLimit;
|
||||
|
||||
/**
|
||||
* 相似度阈值
|
||||
*/
|
||||
private Double similarityThreshold;
|
||||
|
||||
/**
|
||||
* 文本块大小
|
||||
*/
|
||||
@@ -78,6 +83,36 @@ public class KnowledgeInfo extends BaseEntity {
|
||||
*/
|
||||
private String embeddingModel;
|
||||
|
||||
/**
|
||||
* 是否启用重排序(0 否 1是)
|
||||
*/
|
||||
private Integer enableRerank;
|
||||
|
||||
/**
|
||||
* 重排序模型名称
|
||||
*/
|
||||
private String rerankModel;
|
||||
|
||||
/**
|
||||
* 重排序后返回的文档数量
|
||||
*/
|
||||
private Integer rerankTopN;
|
||||
|
||||
/**
|
||||
* 重排序相关性分数阈值
|
||||
*/
|
||||
private Double rerankScoreThreshold;
|
||||
|
||||
/**
|
||||
* 是否启用混合检索(0 否 1是)
|
||||
*/
|
||||
private Integer enableHybrid;
|
||||
|
||||
/**
|
||||
* 混合检索权重 (0.0-1.0)
|
||||
*/
|
||||
private Double hybridAlpha;
|
||||
|
||||
/**
|
||||
* 备注
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package org.ruoyi.domain.vo.knowledge;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 文档分块数统计 VO(用于 GROUP BY 查询结果接收)
|
||||
*/
|
||||
@Data
|
||||
public class DocFragmentCountVo {
|
||||
|
||||
/**
|
||||
* 文档ID(关联 knowledge_attach.doc_id)
|
||||
*/
|
||||
private String docId;
|
||||
|
||||
/**
|
||||
* 该文档下的分块数量
|
||||
*/
|
||||
private Integer fragmentCount;
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import org.ruoyi.domain.entity.knowledge.KnowledgeAttach;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
|
||||
|
||||
@@ -68,5 +69,22 @@ public class KnowledgeAttachVo implements Serializable {
|
||||
@ExcelProperty(value = "备注")
|
||||
private String remark;
|
||||
|
||||
/**
|
||||
* 上传时间(来自 BaseEntity.createTime)
|
||||
*/
|
||||
@ExcelProperty(value = "上传时间")
|
||||
private Date createTime;
|
||||
|
||||
/**
|
||||
* 解析状态: 0待解析, 1解析中, 2已解析, 3解析失败
|
||||
*/
|
||||
@ExcelProperty(value = "解析状态")
|
||||
private Integer status;
|
||||
|
||||
/**
|
||||
* 分块数(统计字段,非数据库列)
|
||||
*/
|
||||
private Integer fragmentCount;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ public class KnowledgeFragmentVo implements Serializable {
|
||||
* 片段索引下标
|
||||
*/
|
||||
@ExcelProperty(value = "片段索引下标")
|
||||
private Long idx;
|
||||
private Integer idx;
|
||||
|
||||
/**
|
||||
* 文档内容
|
||||
@@ -53,5 +53,10 @@ public class KnowledgeFragmentVo implements Serializable {
|
||||
@ExcelProperty(value = "备注")
|
||||
private String remark;
|
||||
|
||||
/**
|
||||
* 知识库ID
|
||||
*/
|
||||
private Long knowledgeId;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -76,6 +76,12 @@ public class KnowledgeInfoVo implements Serializable {
|
||||
@ExcelProperty(value = "知识库中检索的条数")
|
||||
private Integer retrieveLimit;
|
||||
|
||||
/**
|
||||
* 相似度阈值
|
||||
*/
|
||||
@ExcelProperty(value = "相似度阈值")
|
||||
private Double similarityThreshold;
|
||||
|
||||
/**
|
||||
* 文本块大小
|
||||
*/
|
||||
@@ -94,6 +100,48 @@ public class KnowledgeInfoVo implements Serializable {
|
||||
@ExcelProperty(value = "向量模型")
|
||||
private String embeddingModel;
|
||||
|
||||
/**
|
||||
* 是否启用重排序(0 否 1是)
|
||||
*/
|
||||
@ExcelProperty(value = "是否启用重排序")
|
||||
private Integer enableRerank;
|
||||
|
||||
/**
|
||||
* 重排序模型名称
|
||||
*/
|
||||
@ExcelProperty(value = "重排序模型")
|
||||
private String rerankModel;
|
||||
|
||||
/**
|
||||
* 重排序后返回的文档数量
|
||||
*/
|
||||
@ExcelProperty(value = "重排序返回数量")
|
||||
private Integer rerankTopN;
|
||||
|
||||
/**
|
||||
* 重排序相关性分数阈值
|
||||
*/
|
||||
@ExcelProperty(value = "重排序分数阈值")
|
||||
private Double rerankScoreThreshold;
|
||||
|
||||
/**
|
||||
* 是否启用混合检索(0 否 1是)
|
||||
*/
|
||||
@ExcelProperty(value = "是否启用混合检索")
|
||||
private Integer enableHybrid;
|
||||
|
||||
/**
|
||||
* 混合检索权重 (0.0-1.0)
|
||||
*/
|
||||
@ExcelProperty(value = "混合检索权重")
|
||||
private Double hybridAlpha;
|
||||
|
||||
/**
|
||||
* 文档数量
|
||||
*/
|
||||
@ExcelProperty(value = "文档数量")
|
||||
private Integer documentCount;
|
||||
|
||||
/**
|
||||
* 备注
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package org.ruoyi.domain.vo.knowledge;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* 知识检索测试结果视图对象
|
||||
*
|
||||
* @author RobustH
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class KnowledgeRetrievalVo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 片段ID
|
||||
*/
|
||||
private String id;
|
||||
|
||||
/**
|
||||
* 文档ID
|
||||
*/
|
||||
private String docId;
|
||||
|
||||
/**
|
||||
* 知识库ID
|
||||
*/
|
||||
private Long knowledgeId;
|
||||
|
||||
/**
|
||||
* 分片索引
|
||||
*/
|
||||
private Integer idx;
|
||||
|
||||
/**
|
||||
* 片段内容
|
||||
*/
|
||||
private String content;
|
||||
|
||||
/**
|
||||
* 相似度得分
|
||||
*/
|
||||
private Double score;
|
||||
|
||||
/**
|
||||
* 原始检索排名 (重排前)
|
||||
*/
|
||||
private Integer originalIndex;
|
||||
|
||||
/**
|
||||
* 原始检索得分 (重排前)
|
||||
*/
|
||||
private Double rawScore;
|
||||
|
||||
/**
|
||||
* 来源文档名称
|
||||
*/
|
||||
private String sourceName;
|
||||
}
|
||||
@@ -18,8 +18,7 @@ public enum ChatModeType {
|
||||
PPIO("ppio", "ppio"),
|
||||
CUSTOM_API("custom_api", "自定义API"),
|
||||
MINIMAX("minimax", "MiniMax"),
|
||||
XIAOMI("xiaomi", "小米MiMo"),
|
||||
DIFY("dify", "Dify平台");
|
||||
XIAOMI("xiaomi", "小米MiMo");
|
||||
private final String code;
|
||||
private final String description;
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package org.ruoyi.enums;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* 知识库附件解析状态枚举
|
||||
*
|
||||
* @author RobustH
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum KnowledgeAttachStatus {
|
||||
|
||||
/**
|
||||
* 待解析
|
||||
*/
|
||||
WAITING(0, "待解析"),
|
||||
|
||||
/**
|
||||
* 解析中
|
||||
*/
|
||||
PARSING(1, "解析中"),
|
||||
|
||||
/**
|
||||
* 已解析
|
||||
*/
|
||||
COMPLETED(2, "已解析"),
|
||||
|
||||
/**
|
||||
* 解析失败
|
||||
*/
|
||||
FAILED(3, "解析失败");
|
||||
|
||||
private final Integer code;
|
||||
private final String info;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package org.ruoyi.factory;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.common.chat.service.chat.IChatModelService;
|
||||
import org.ruoyi.service.rerank.RerankModelService;
|
||||
import org.springframework.beans.factory.NoSuchBeanDefinitionException;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* 重排序模型工厂服务类
|
||||
* 参考设计模式:EmbeddingModelFactory
|
||||
* 负责创建和管理重排序模型实例
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-19
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class RerankModelFactory {
|
||||
|
||||
private final ApplicationContext applicationContext;
|
||||
|
||||
private final IChatModelService chatModelService;
|
||||
|
||||
/**
|
||||
* 模型缓存,使用ConcurrentHashMap保证线程安全
|
||||
*/
|
||||
private final Map<String, RerankModelService> modelCache = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 创建重排序模型实例
|
||||
* 如果模型已存在于缓存中,则直接返回;否则创建新的实例
|
||||
*
|
||||
* @param rerankModelName 重排序模型名称
|
||||
*/
|
||||
public RerankModelService createModel(String rerankModelName) {
|
||||
return modelCache.computeIfAbsent(rerankModelName, name -> {
|
||||
ChatModelVo modelConfig = chatModelService.selectModelByName(rerankModelName);
|
||||
|
||||
if (modelConfig == null) {
|
||||
throw new IllegalArgumentException("未找到重排序模型配置,name=" + name);
|
||||
}
|
||||
return createModelInstance(modelConfig.getProviderCode(), modelConfig);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新模型缓存
|
||||
* 根据给定的模型ID从缓存中移除对应的模型
|
||||
*
|
||||
* @param modelId 模型的唯一标识ID
|
||||
*/
|
||||
public void refreshModel(Long modelId) {
|
||||
modelCache.remove(modelId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有支持模型工厂的列表
|
||||
*
|
||||
* @return 支持的模型工厂名称列表
|
||||
*/
|
||||
public List<String> getSupportedFactories() {
|
||||
return new ArrayList<>(applicationContext.getBeansOfType(RerankModelService.class)
|
||||
.keySet());
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建具体的模型实例
|
||||
* 根据提供的工厂名称和配置信息创建并配置模型实例
|
||||
*
|
||||
* @param factory 工厂名称,用于标识模型类型(providerCode)
|
||||
* @param config 模型配置信息
|
||||
* @return RerankModelService 配置好的模型实例
|
||||
* @throws IllegalArgumentException 当无法获取指定的模型实例时抛出
|
||||
*/
|
||||
private RerankModelService createModelInstance(String factory, ChatModelVo config) {
|
||||
try {
|
||||
// 优先尝试使用 providerCode + "Rerank" 作为 Bean 名称
|
||||
// 例如:zhipu -> zhipuRerank,jina -> jinaRerank
|
||||
String rerankBeanName = factory + "Rerank";
|
||||
RerankModelService model = applicationContext.getBean(rerankBeanName, RerankModelService.class);
|
||||
model.configure(config);
|
||||
log.info("成功创建重排序模型: factory={}, modelName={}", rerankBeanName, config.getModelName());
|
||||
return model;
|
||||
} catch (NoSuchBeanDefinitionException e) {
|
||||
// 如果找不到,尝试使用原始的 providerCode
|
||||
try {
|
||||
RerankModelService model = applicationContext.getBean(factory, RerankModelService.class);
|
||||
model.configure(config);
|
||||
log.info("成功创建重排序模型: factory={}, modelName={}", factory, config.getModelName());
|
||||
return model;
|
||||
} catch (NoSuchBeanDefinitionException ex) {
|
||||
throw new IllegalArgumentException("获取不到重排序模型: " + factory + " 或 " + factory + "Rerank", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
package org.ruoyi.mapper.knowledge;
|
||||
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.ruoyi.domain.entity.knowledge.KnowledgeAttach;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
|
||||
import org.ruoyi.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
@@ -10,6 +13,12 @@ import org.ruoyi.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
* @author ageerle
|
||||
* @date 2025-12-17
|
||||
*/
|
||||
@Mapper
|
||||
public interface KnowledgeAttachMapper extends BaseMapperPlus<KnowledgeAttach, KnowledgeAttachVo> {
|
||||
|
||||
/**
|
||||
* 统计指定知识库下的文档数量
|
||||
*/
|
||||
@Select("SELECT COUNT(*) FROM knowledge_attach WHERE knowledge_id = #{knowledgeId}")
|
||||
int countByKnowledgeId(@Param("knowledgeId") Long knowledgeId);
|
||||
}
|
||||
|
||||
@@ -1,15 +1,45 @@
|
||||
package org.ruoyi.mapper.knowledge;
|
||||
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.ruoyi.domain.entity.knowledge.KnowledgeFragment;
|
||||
import org.ruoyi.domain.vo.knowledge.DocFragmentCountVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeFragmentVo;
|
||||
import org.ruoyi.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 知识片段Mapper接口
|
||||
*
|
||||
* @author ageerle
|
||||
* @date 2025-12-17
|
||||
*/
|
||||
@Mapper
|
||||
public interface KnowledgeFragmentMapper extends BaseMapperPlus<KnowledgeFragment, KnowledgeFragmentVo> {
|
||||
|
||||
/**
|
||||
* 批量统计各文档的分块数(强类型接收,避免 Map key 大小写问题)
|
||||
*
|
||||
* @param docIds 文档 ID 列表
|
||||
* @return 每个 docId 对应的分块数列表
|
||||
*/
|
||||
@Select("<script>" +
|
||||
"SELECT doc_id AS docId, COUNT(*) AS fragmentCount " +
|
||||
"FROM knowledge_fragment " +
|
||||
"WHERE doc_id IN " +
|
||||
"<foreach collection='docIds' item='id' open='(' separator=',' close=')'>#{id}</foreach> " +
|
||||
"GROUP BY doc_id" +
|
||||
"</script>")
|
||||
List<DocFragmentCountVo> selectFragmentCountByDocIds(@Param("docIds") List<String> docIds);
|
||||
@Select("<script>" +
|
||||
"SELECT id, doc_id AS docId, content, idx, knowledge_id AS knowledgeId " +
|
||||
"FROM knowledge_fragment " +
|
||||
"WHERE knowledge_id = #{knowledgeId} " +
|
||||
"AND MATCH (content) AGAINST (#{query} IN NATURAL LANGUAGE MODE) " +
|
||||
"ORDER BY MATCH (content) AGAINST (#{query} IN NATURAL LANGUAGE MODE) DESC " +
|
||||
"LIMIT #{limit}" +
|
||||
"</script>")
|
||||
List<KnowledgeFragmentVo> searchByKeyword(@Param("knowledgeId") Long knowledgeId, @Param("query") String query, @Param("limit") Integer limit);
|
||||
}
|
||||
|
||||
@@ -20,6 +20,11 @@ import dev.langchain4j.model.chat.response.StreamingChatResponseHandler;
|
||||
import dev.langchain4j.model.openai.OpenAiChatModel;
|
||||
import dev.langchain4j.service.tool.ToolProvider;
|
||||
import dev.langchain4j.skills.shell.ShellSkills;
|
||||
import dev.langchain4j.rag.AugmentationRequest;
|
||||
import dev.langchain4j.rag.AugmentationResult;
|
||||
import dev.langchain4j.rag.DefaultRetrievalAugmentor;
|
||||
import dev.langchain4j.rag.RetrievalAugmentor;
|
||||
import dev.langchain4j.rag.query.Metadata;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -47,7 +52,6 @@ import org.ruoyi.common.sse.core.SseEmitterManager;
|
||||
import org.ruoyi.common.sse.utils.SseMessageUtils;
|
||||
import org.ruoyi.domain.bo.vector.QueryVectorBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
|
||||
import org.ruoyi.enums.ChatModeType;
|
||||
import org.ruoyi.factory.ChatServiceFactory;
|
||||
import org.ruoyi.mcp.service.core.ToolProviderFactory;
|
||||
import org.ruoyi.observability.*;
|
||||
@@ -55,6 +59,8 @@ import org.ruoyi.service.chat.AbstractChatService;
|
||||
import org.ruoyi.service.chat.IChatMessageService;
|
||||
import org.ruoyi.service.chat.impl.memory.PersistentChatMemoryStore;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
|
||||
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
|
||||
import org.ruoyi.service.knowledge.retriever.CustomVectorRetriever;
|
||||
import org.ruoyi.service.vector.VectorStoreService;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
@@ -90,6 +96,8 @@ public class ChatServiceFacade implements IChatService {
|
||||
|
||||
private final VectorStoreService vectorStoreService;
|
||||
|
||||
private final KnowledgeRetrievalService knowledgeRetrievalService;
|
||||
|
||||
private final SseEmitterManager sseEmitterManager;
|
||||
|
||||
private final IChatMessageService chatMessageService;
|
||||
@@ -98,8 +106,6 @@ public class ChatServiceFacade implements IChatService {
|
||||
|
||||
private final ToolProviderFactory toolProviderFactory;
|
||||
|
||||
private final org.ruoyi.service.chat.impl.provider.DifyWorkflowService difyWorkflowService;
|
||||
|
||||
/**
|
||||
* 内存实例缓存,避免同一会话重复创建
|
||||
* Key: sessionId, Value: MessageWindowChatMemory实例
|
||||
@@ -166,14 +172,6 @@ public class ChatServiceFacade implements IChatService {
|
||||
* @return 如果需要提前返回则返回SseEmitter,否则返回null
|
||||
*/
|
||||
private SseEmitter handleSpecialChatModes(ChatRequest chatRequest) {
|
||||
// 处理 Dify 工作流对话
|
||||
if (chatRequest.getEnableWorkFlow()
|
||||
&& chatRequest.getChatModelVo() != null
|
||||
&& ChatModeType.DIFY.getCode().equals(chatRequest.getChatModelVo().getProviderCode())) {
|
||||
log.info("处理Dify工作流对话,会话: {}", chatRequest.getSessionId());
|
||||
return difyWorkflowService.streaming(chatRequest.getChatModelVo(), chatRequest);
|
||||
}
|
||||
|
||||
// 处理工作流对话
|
||||
if (chatRequest.getEnableWorkFlow()) {
|
||||
log.info("处理工作流对话,会话: {}", chatRequest.getSessionId());
|
||||
@@ -420,16 +418,49 @@ public class ChatServiceFacade implements IChatService {
|
||||
|
||||
/**
|
||||
* 构建上下文消息列表
|
||||
|
||||
* 消息顺序:历史消息 → 当前用户消息(确保 AI 正确理解对话上下文)
|
||||
*
|
||||
* @param chatRequest 聊天请求
|
||||
* @return 上下文消息列表
|
||||
*/
|
||||
private List<ChatMessage> buildContextMessages(ChatRequest chatRequest) {
|
||||
List<ChatMessage> messages = new ArrayList<>();
|
||||
List<ChatMessage> messages = new ArrayList<>();
|
||||
|
||||
// 从数据库查询历史对话消息(放在前面)
|
||||
// 1. 初始化当前用户消息
|
||||
UserMessage userMessage = UserMessage.userMessage(chatRequest.getContent());
|
||||
|
||||
// 2. 知识库检索增强 (RAG)
|
||||
if (chatRequest.getKnowledgeId() != null) {
|
||||
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(chatRequest.getKnowledgeId()));
|
||||
if (knowledgeInfoVo != null) {
|
||||
ChatModelVo chatModel = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel());
|
||||
if (chatModel != null) {
|
||||
log.info("执行高级 RAG 流程: kid={}", chatRequest.getKnowledgeId());
|
||||
|
||||
// 构建自定义检索器
|
||||
CustomVectorRetriever retriever = new CustomVectorRetriever(
|
||||
knowledgeRetrievalService, knowledgeInfoVo, chatModel);
|
||||
|
||||
// 构建增强流水线
|
||||
RetrievalAugmentor augmentor = DefaultRetrievalAugmentor.builder()
|
||||
.contentRetriever(retriever)
|
||||
.build();
|
||||
|
||||
// 执行增强:编织上下文到 UserMessage
|
||||
Metadata metadata = Metadata.from(userMessage, chatRequest.getSessionId(), new ArrayList<>());
|
||||
AugmentationRequest augmentationRequest = new AugmentationRequest(userMessage, metadata);
|
||||
AugmentationResult result = augmentor.augment(augmentationRequest);
|
||||
|
||||
ChatMessage augmented = result.chatMessage();
|
||||
if (augmented instanceof UserMessage) {
|
||||
userMessage = (UserMessage) augmented;
|
||||
log.debug("RAG 增强完成,UserMessage 已注入背景知识");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 从数据库查询历史对话消息(放在前面)
|
||||
if (chatRequest.getSessionId() != null) {
|
||||
MessageWindowChatMemory memory = createChatMemory(chatRequest.getSessionId());
|
||||
if (memory != null) {
|
||||
@@ -441,42 +472,7 @@ public class ChatServiceFacade implements IChatService {
|
||||
}
|
||||
}
|
||||
|
||||
// Dify 自带 RAG 知识库检索,跳过本地向量库查询
|
||||
boolean isDifyProvider = chatRequest.getChatModelVo() != null
|
||||
&& ChatModeType.DIFY.getCode().equals(chatRequest.getChatModelVo().getProviderCode());
|
||||
|
||||
// 从向量库查询相关历史消息(知识库内容作为上下文)
|
||||
if (chatRequest.getKnowledgeId() != null && !isDifyProvider) {
|
||||
// 查询知识库信息
|
||||
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(chatRequest.getKnowledgeId()));
|
||||
if (knowledgeInfoVo == null) {
|
||||
log.warn("知识库信息不存在,kid: {}", chatRequest.getKnowledgeId());
|
||||
// 继续添加当前用户消息
|
||||
messages.add(UserMessage.userMessage(chatRequest.getContent()));
|
||||
return messages;
|
||||
}
|
||||
|
||||
// 查询向量模型配置信息
|
||||
ChatModelVo chatModel = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel());
|
||||
if (chatModel == null) {
|
||||
log.warn("向量模型配置不存在,模型名称: {}", knowledgeInfoVo.getEmbeddingModel());
|
||||
messages.add(UserMessage.userMessage(chatRequest.getContent()));
|
||||
return messages;
|
||||
}
|
||||
|
||||
// 构建向量查询参数
|
||||
QueryVectorBo queryVectorBo = buildQueryVectorBo(chatRequest, knowledgeInfoVo, chatModel);
|
||||
|
||||
// 获取向量查询结果(知识库内容作为系统上下文,放在历史消息之后)
|
||||
List<String> nearestList = vectorStoreService.getQueryVector(queryVectorBo);
|
||||
for (String prompt : nearestList) {
|
||||
// 知识库内容作为系统上下文添加
|
||||
messages.add(new AiMessage(prompt));
|
||||
}
|
||||
}
|
||||
|
||||
// 构建当前用户消息(放在最后)
|
||||
UserMessage userMessage = UserMessage.userMessage(chatRequest.getContent());
|
||||
// 4. 添加经过增强的用户消息(放在最后)
|
||||
messages.add(userMessage);
|
||||
|
||||
return messages;
|
||||
@@ -495,6 +491,13 @@ public class ChatServiceFacade implements IChatService {
|
||||
queryVectorBo.setVectorModelName(knowledgeInfoVo.getVectorModel());
|
||||
queryVectorBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModel());
|
||||
queryVectorBo.setMaxResults(knowledgeInfoVo.getRetrieveLimit());
|
||||
|
||||
// 设置重排序参数
|
||||
queryVectorBo.setEnableRerank(knowledgeInfoVo.getEnableRerank() != null && knowledgeInfoVo.getEnableRerank() == 1);
|
||||
queryVectorBo.setRerankModelName(knowledgeInfoVo.getRerankModel());
|
||||
queryVectorBo.setRerankTopN(knowledgeInfoVo.getRerankTopN());
|
||||
queryVectorBo.setRerankScoreThreshold(knowledgeInfoVo.getRerankScoreThreshold());
|
||||
|
||||
return queryVectorBo;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
package org.ruoyi.service.chat.impl.provider;
|
||||
|
||||
import dev.langchain4j.model.chat.ChatModel;
|
||||
import dev.langchain4j.model.chat.StreamingChatModel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.common.chat.domain.dto.request.ChatRequest;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.enums.ChatModeType;
|
||||
import org.ruoyi.service.chat.AbstractChatService;
|
||||
import org.ruoyi.service.chat.impl.provider.model.DifyStreamingChatModel;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* Dify 平台对话服务
|
||||
* <p>
|
||||
* 通过 dify-java-client 接入 Dify 的对话型应用 (Chat App) 和
|
||||
* 工作流编排对话应用 (Chatflow App),支持流式 SSE 响应。
|
||||
*
|
||||
* @author better
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class DifyChatServiceImpl implements AbstractChatService {
|
||||
|
||||
private final DifyConversationService difyConversationService;
|
||||
|
||||
@Override
|
||||
public StreamingChatModel buildStreamingChatModel(ChatModelVo chatModelVo, ChatRequest chatRequest) {
|
||||
return new DifyStreamingChatModel(chatModelVo, chatRequest, difyConversationService);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatModel buildChatModel(ChatModelVo chatModelVo) {
|
||||
throw new UnsupportedOperationException("Dify 不支持同步 ChatModel,请使用流式模式");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getProviderName() {
|
||||
return ChatModeType.DIFY.getCode();
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package org.ruoyi.service.chat.impl.provider;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Dify 会话映射管理
|
||||
* <p>
|
||||
* 维护 ruoyi sessionId 与 Dify conversation_id 的映射关系,
|
||||
* 确保多轮对话上下文连续。
|
||||
*
|
||||
* @author better
|
||||
*/
|
||||
@Service
|
||||
public class DifyConversationService {
|
||||
|
||||
private final ConcurrentHashMap<Long, String> conversationMap = new ConcurrentHashMap<>();
|
||||
|
||||
public String getConversationId(Long sessionId) {
|
||||
return conversationMap.get(sessionId);
|
||||
}
|
||||
|
||||
public void saveMapping(Long sessionId, String difyConversationId) {
|
||||
if (sessionId != null && difyConversationId != null) {
|
||||
conversationMap.put(sessionId, difyConversationId);
|
||||
}
|
||||
}
|
||||
|
||||
public void clearMapping(Long sessionId) {
|
||||
if (sessionId != null) {
|
||||
conversationMap.remove(sessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
package org.ruoyi.service.chat.impl.provider;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import io.github.imfangs.dify.client.DifyClientFactory;
|
||||
import io.github.imfangs.dify.client.DifyWorkflowClient;
|
||||
import io.github.imfangs.dify.client.enums.ResponseMode;
|
||||
import io.github.imfangs.dify.client.event.ErrorEvent;
|
||||
import io.github.imfangs.dify.client.event.WorkflowFinishedEvent;
|
||||
import io.github.imfangs.dify.client.event.WorkflowTextChunkEvent;
|
||||
import io.github.imfangs.dify.client.callback.WorkflowStreamCallback;
|
||||
import io.github.imfangs.dify.client.model.workflow.WorkflowRunRequest;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.common.chat.domain.dto.request.ChatRequest;
|
||||
import org.ruoyi.common.chat.domain.dto.request.WorkFlowRunner;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.common.sse.utils.SseMessageUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
/**
|
||||
* Dify 工作流执行服务
|
||||
* <p>
|
||||
* 通过 DifyWorkflowClient 调用 Dify 平台上部署的工作流应用,
|
||||
* 并将节点事件通过 SSE 实时推送给前端。
|
||||
*
|
||||
* @author better
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class DifyWorkflowService {
|
||||
|
||||
/**
|
||||
* 流式执行 Dify 工作流
|
||||
*
|
||||
* @param chatModelVo 模型配置(apiHost= Dify 地址, apiKey= Dify 密钥)
|
||||
* @param chatRequest 聊天请求
|
||||
* @return SSE emitter
|
||||
*/
|
||||
public SseEmitter streaming(ChatModelVo chatModelVo, ChatRequest chatRequest) {
|
||||
Long userId = chatRequest.getUserId();
|
||||
String tokenValue = chatRequest.getTokenValue();
|
||||
SseEmitter emitter = chatRequest.getEmitter();
|
||||
|
||||
// 构建 Dify 工作流请求参数
|
||||
Map<String, Object> inputs = convertInputs(chatRequest.getWorkFlowRunner());
|
||||
|
||||
WorkflowRunRequest request = WorkflowRunRequest.builder()
|
||||
.inputs(inputs)
|
||||
.responseMode(ResponseMode.STREAMING)
|
||||
.user(String.valueOf(userId))
|
||||
.build();
|
||||
|
||||
DifyWorkflowClient client = DifyClientFactory.createWorkflowClient(
|
||||
normalizeBaseUrl(chatModelVo.getApiHost()),
|
||||
chatModelVo.getApiKey());
|
||||
|
||||
// 异步执行,避免阻塞请求线程
|
||||
CompletableFuture.runAsync(() -> {
|
||||
try {
|
||||
client.runWorkflowStream(request, new WorkflowStreamCallback() {
|
||||
|
||||
@Override
|
||||
public void onWorkflowTextChunk(WorkflowTextChunkEvent event) {
|
||||
String text = event.getData() != null ? event.getData().getText() : null;
|
||||
if (text != null) {
|
||||
SseMessageUtils.sendContent(userId, text);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onWorkflowFinished(WorkflowFinishedEvent event) {
|
||||
// 将最终输出作为内容发送
|
||||
if (event.getData() != null && event.getData().getOutputs() != null) {
|
||||
Map<String, Object> outputs = event.getData().getOutputs();
|
||||
for (Map.Entry<String, Object> entry : outputs.entrySet()) {
|
||||
SseMessageUtils.sendContent(userId,
|
||||
entry.getKey() + ": " + entry.getValue() + "\n");
|
||||
}
|
||||
}
|
||||
SseMessageUtils.sendDone(userId);
|
||||
SseMessageUtils.completeConnection(userId, tokenValue);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(ErrorEvent event) {
|
||||
SseMessageUtils.sendError(userId, event.getMessage());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onException(Throwable throwable) {
|
||||
log.error("Dify 工作流执行异常", throwable);
|
||||
SseMessageUtils.sendError(userId, throwable.getMessage());
|
||||
SseMessageUtils.completeConnection(userId, tokenValue);
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.error("Dify 工作流执行失败", e);
|
||||
SseMessageUtils.sendError(userId, e.getMessage());
|
||||
SseMessageUtils.completeConnection(userId, tokenValue);
|
||||
}
|
||||
});
|
||||
|
||||
return emitter;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 WorkFlowRunner.inputs (List<ObjectNode>) 转换为 Dify 所需的 Map
|
||||
*/
|
||||
private Map<String, Object> convertInputs(WorkFlowRunner runner) {
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
if (runner == null || runner.getInputs() == null) {
|
||||
return result;
|
||||
}
|
||||
for (ObjectNode node : runner.getInputs()) {
|
||||
Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
|
||||
while (fields.hasNext()) {
|
||||
Map.Entry<String, JsonNode> field = fields.next();
|
||||
result.put(field.getKey(), field.getValue().asText());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String normalizeBaseUrl(String baseUrl) {
|
||||
if (baseUrl == null || baseUrl.isBlank()) {
|
||||
throw new IllegalArgumentException("Dify API 地址(apiHost)不能为空");
|
||||
}
|
||||
return baseUrl.endsWith("/") ? baseUrl.substring(0, baseUrl.length() - 1) : baseUrl;
|
||||
}
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
package org.ruoyi.service.chat.impl.provider.model;
|
||||
|
||||
import dev.langchain4j.data.message.AiMessage;
|
||||
import dev.langchain4j.data.message.ChatMessage;
|
||||
import dev.langchain4j.data.message.UserMessage;
|
||||
import dev.langchain4j.model.chat.StreamingChatModel;
|
||||
import dev.langchain4j.model.chat.response.ChatResponse;
|
||||
import dev.langchain4j.model.chat.response.StreamingChatResponseHandler;
|
||||
import io.github.imfangs.dify.client.DifyChatClient;
|
||||
import io.github.imfangs.dify.client.DifyClientFactory;
|
||||
import io.github.imfangs.dify.client.enums.ResponseMode;
|
||||
import io.github.imfangs.dify.client.event.ErrorEvent;
|
||||
import io.github.imfangs.dify.client.event.MessageEndEvent;
|
||||
import io.github.imfangs.dify.client.event.MessageEvent;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.common.chat.domain.dto.request.ChatRequest;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.service.chat.impl.provider.DifyConversationService;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Dify 流式聊天模型适配器
|
||||
* <p>
|
||||
* 将 Dify 的回调式流式响应适配为 langchain4j 的 StreamingChatModel 接口,
|
||||
* 使 ChatServiceFacade 可以像其他 provider 一样统一调用。
|
||||
*
|
||||
* @author better
|
||||
*/
|
||||
@Slf4j
|
||||
public class DifyStreamingChatModel implements StreamingChatModel {
|
||||
|
||||
private final ChatModelVo chatModelVo;
|
||||
private final ChatRequest chatRequest;
|
||||
private final DifyConversationService conversationService;
|
||||
|
||||
public DifyStreamingChatModel(ChatModelVo chatModelVo, ChatRequest chatRequest,
|
||||
DifyConversationService conversationService) {
|
||||
this.chatModelVo = chatModelVo;
|
||||
this.chatRequest = chatRequest;
|
||||
this.conversationService = conversationService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void chat(List<ChatMessage> messages, StreamingChatResponseHandler handler) {
|
||||
// 1. 从 langchain4j 消息列表中提取最后一条用户消息作为 query
|
||||
String query = extractUserQuery(messages);
|
||||
|
||||
// 2. 获取 Dify conversation_id(多轮对话连续性)
|
||||
String conversationId = null;
|
||||
if (chatRequest.getSessionId() != null) {
|
||||
conversationId = conversationService.getConversationId(chatRequest.getSessionId());
|
||||
}
|
||||
|
||||
// 3. 构建 Dify 请求
|
||||
io.github.imfangs.dify.client.model.chat.ChatMessage difyMessage = io.github.imfangs.dify.client.model.chat.ChatMessage.builder()
|
||||
.query(query)
|
||||
.user(String.valueOf(chatRequest.getUserId()))
|
||||
.responseMode(ResponseMode.STREAMING)
|
||||
.conversationId(conversationId)
|
||||
.autoGenerateName(true)
|
||||
.build();
|
||||
|
||||
// 4. 创建 Dify 客户端并发送流式请求
|
||||
try {
|
||||
DifyChatClient client = DifyClientFactory.createChatClient(
|
||||
normalizeBaseUrl(chatModelVo.getApiHost()),
|
||||
chatModelVo.getApiKey());
|
||||
|
||||
client.sendChatMessageStream(difyMessage, new DifyChatStreamAdapter(handler));
|
||||
} catch (Exception e) {
|
||||
log.error("Dify 流式对话调用失败", e);
|
||||
handler.onError(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void chat(String userMessage, StreamingChatResponseHandler handler) {
|
||||
io.github.imfangs.dify.client.model.chat.ChatMessage difyMessage = io.github.imfangs.dify.client.model.chat.ChatMessage.builder()
|
||||
.query(userMessage)
|
||||
.user(String.valueOf(chatRequest.getUserId()))
|
||||
.responseMode(ResponseMode.STREAMING)
|
||||
.conversationId(chatRequest.getSessionId() != null
|
||||
? conversationService.getConversationId(chatRequest.getSessionId()) : null)
|
||||
.autoGenerateName(true)
|
||||
.build();
|
||||
|
||||
try {
|
||||
DifyChatClient client = DifyClientFactory.createChatClient(
|
||||
normalizeBaseUrl(chatModelVo.getApiHost()),
|
||||
chatModelVo.getApiKey());
|
||||
|
||||
client.sendChatMessageStream(difyMessage, new DifyChatStreamAdapter(handler));
|
||||
} catch (Exception e) {
|
||||
log.error("Dify 流式对话调用失败", e);
|
||||
handler.onError(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 langchain4j 消息列表中提取最后一条用户消息文本
|
||||
*/
|
||||
private String extractUserQuery(List<ChatMessage> messages) {
|
||||
for (int i = messages.size() - 1; i >= 0; i--) {
|
||||
ChatMessage msg = messages.get(i);
|
||||
if (msg instanceof UserMessage) {
|
||||
return ((UserMessage) msg).singleText();
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private String normalizeBaseUrl(String baseUrl) {
|
||||
if (baseUrl == null || baseUrl.isBlank()) {
|
||||
throw new IllegalArgumentException("Dify API 地址(apiHost)不能为空");
|
||||
}
|
||||
return baseUrl.endsWith("/") ? baseUrl.substring(0, baseUrl.length() - 1) : baseUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dify 回调适配器
|
||||
* 将 Dify ChatStreamCallback 事件转发给 langchain4j StreamingChatResponseHandler
|
||||
*/
|
||||
private class DifyChatStreamAdapter implements io.github.imfangs.dify.client.callback.ChatStreamCallback {
|
||||
|
||||
private final StreamingChatResponseHandler handler;
|
||||
private final StringBuilder fullResponse = new StringBuilder();
|
||||
|
||||
DifyChatStreamAdapter(StreamingChatResponseHandler handler) {
|
||||
this.handler = handler;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onMessage(MessageEvent event) {
|
||||
String answer = event.getAnswer();
|
||||
if (answer != null) {
|
||||
fullResponse.append(answer);
|
||||
handler.onPartialResponse(answer);
|
||||
}
|
||||
// 保存 Dify conversation_id 以维持多轮对话
|
||||
if (event.getConversationId() != null && chatRequest.getSessionId() != null) {
|
||||
conversationService.saveMapping(chatRequest.getSessionId(), event.getConversationId());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onMessageEnd(MessageEndEvent event) {
|
||||
// 保存 conversation_id
|
||||
if (event.getConversationId() != null && chatRequest.getSessionId() != null) {
|
||||
conversationService.saveMapping(chatRequest.getSessionId(), event.getConversationId());
|
||||
}
|
||||
|
||||
// 构建完整的 ChatResponse 交给上层处理
|
||||
AiMessage aiMessage = new AiMessage(fullResponse.toString());
|
||||
ChatResponse response = ChatResponse.builder()
|
||||
.aiMessage(aiMessage)
|
||||
.id(event.getMessageId())
|
||||
.build();
|
||||
handler.onCompleteResponse(response);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(ErrorEvent event) {
|
||||
handler.onError(new RuntimeException(event.getMessage()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onException(Throwable throwable) {
|
||||
handler.onError(throwable);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -72,4 +72,11 @@ public interface IKnowledgeAttachService {
|
||||
* 上传附件
|
||||
*/
|
||||
void upload(KnowledgeInfoUploadBo bo);
|
||||
|
||||
/**
|
||||
* 解析附件知识片段
|
||||
*
|
||||
* @param id 附件ID
|
||||
*/
|
||||
void parse(Long id);
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import org.ruoyi.common.mybatis.core.page.TableDataInfo;
|
||||
import org.ruoyi.common.mybatis.core.page.PageQuery;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeFragmentBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeFragmentVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
@@ -65,4 +66,12 @@ public interface IKnowledgeFragmentService {
|
||||
* @return 是否删除成功
|
||||
*/
|
||||
Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid);
|
||||
|
||||
/**
|
||||
* 检索测试
|
||||
*
|
||||
* @param bo 检索参数
|
||||
* @return 检索结果
|
||||
*/
|
||||
List<KnowledgeRetrievalVo> retrieval(KnowledgeFragmentBo bo);
|
||||
}
|
||||
|
||||
@@ -2,24 +2,27 @@ package org.ruoyi.service.knowledge.impl;
|
||||
|
||||
import cn.hutool.core.collection.CollUtil;
|
||||
import cn.hutool.core.util.RandomUtil;
|
||||
import org.ruoyi.common.chat.service.chat.IChatModelService;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.common.chat.service.chat.IChatModelService;
|
||||
import org.ruoyi.enums.KnowledgeAttachStatus;
|
||||
import org.ruoyi.common.core.domain.dto.OssDTO;
|
||||
import org.ruoyi.common.core.service.OssService;
|
||||
import org.ruoyi.common.core.utils.MapstructUtils;
|
||||
import org.ruoyi.common.core.utils.SpringUtils;
|
||||
import org.ruoyi.common.core.utils.StringUtils;
|
||||
import org.ruoyi.common.mybatis.core.page.TableDataInfo;
|
||||
import org.ruoyi.common.mybatis.core.page.PageQuery;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.common.mybatis.core.page.TableDataInfo;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo;
|
||||
import org.ruoyi.domain.bo.vector.StoreEmbeddingBo;
|
||||
import org.ruoyi.domain.entity.knowledge.KnowledgeAttach;
|
||||
import org.ruoyi.domain.entity.knowledge.KnowledgeFragment;
|
||||
import org.ruoyi.domain.vo.knowledge.DocFragmentCountVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
|
||||
import org.ruoyi.factory.ResourceLoaderFactory;
|
||||
@@ -29,11 +32,15 @@ import org.ruoyi.service.knowledge.IKnowledgeAttachService;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
|
||||
import org.ruoyi.service.knowledge.ResourceLoader;
|
||||
import org.ruoyi.service.vector.VectorStoreService;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import java.net.URL;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 知识库附件Service业务层处理
|
||||
@@ -47,57 +54,51 @@ import java.util.*;
|
||||
public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
|
||||
private final KnowledgeAttachMapper baseMapper;
|
||||
|
||||
private final IKnowledgeInfoService knowledgeInfoService;
|
||||
|
||||
private final KnowledgeFragmentMapper knowledgeFragmentMapper;
|
||||
|
||||
private final IChatModelService chatModelService;
|
||||
|
||||
private final ResourceLoaderFactory resourceLoaderFactory;
|
||||
|
||||
private final VectorStoreService vectorStoreService;
|
||||
|
||||
private final OssService ossService;
|
||||
/**
|
||||
* 查询知识库附件
|
||||
*
|
||||
* @param id 主键
|
||||
* @return 知识库附件
|
||||
*/
|
||||
|
||||
@Override
|
||||
public KnowledgeAttachVo queryById(Long id){
|
||||
public KnowledgeAttachVo queryById(Long id) {
|
||||
return baseMapper.selectVoById(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页查询知识库附件列表
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @param pageQuery 分页参数
|
||||
* @return 知识库附件分页列表
|
||||
*/
|
||||
@Override
|
||||
public TableDataInfo<KnowledgeAttachVo> queryPageList(KnowledgeAttachBo bo, PageQuery pageQuery) {
|
||||
LambdaQueryWrapper<KnowledgeAttach> lqw = buildQueryWrapper(bo);
|
||||
Page<KnowledgeAttachVo> result = baseMapper.selectVoPage(pageQuery.build(), lqw);
|
||||
fillFragmentCount(result.getRecords());
|
||||
return TableDataInfo.build(result);
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询符合条件的知识库附件列表
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @return 知识库附件列表
|
||||
*/
|
||||
@Override
|
||||
public List<KnowledgeAttachVo> queryList(KnowledgeAttachBo bo) {
|
||||
LambdaQueryWrapper<KnowledgeAttach> lqw = buildQueryWrapper(bo);
|
||||
return baseMapper.selectVoList(lqw);
|
||||
List<KnowledgeAttachVo> list = baseMapper.selectVoList(lqw);
|
||||
fillFragmentCount(list);
|
||||
return list;
|
||||
}
|
||||
|
||||
private void fillFragmentCount(List<KnowledgeAttachVo> records) {
|
||||
if (records == null || records.isEmpty()) return;
|
||||
List<String> docIds = records.stream()
|
||||
.map(KnowledgeAttachVo::getDocId)
|
||||
.filter(StringUtils::isNotBlank)
|
||||
.distinct()
|
||||
.collect(Collectors.toList());
|
||||
if (docIds.isEmpty()) return;
|
||||
List<DocFragmentCountVo> countList = knowledgeFragmentMapper.selectFragmentCountByDocIds(docIds);
|
||||
Map<String, Integer> countMap = countList.stream()
|
||||
.collect(Collectors.toMap(DocFragmentCountVo::getDocId, DocFragmentCountVo::getFragmentCount, (k1, k2) -> k1));
|
||||
for (KnowledgeAttachVo vo : records) {
|
||||
vo.setFragmentCount(countMap.getOrDefault(vo.getDocId(), 0));
|
||||
}
|
||||
}
|
||||
|
||||
private LambdaQueryWrapper<KnowledgeAttach> buildQueryWrapper(KnowledgeAttachBo bo) {
|
||||
Map<String, Object> params = bo.getParams();
|
||||
LambdaQueryWrapper<KnowledgeAttach> lqw = Wrappers.lambdaQuery();
|
||||
lqw.orderByAsc(KnowledgeAttach::getId);
|
||||
lqw.eq(bo.getKnowledgeId() != null, KnowledgeAttach::getKnowledgeId, bo.getKnowledgeId());
|
||||
@@ -107,16 +108,9 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
return lqw;
|
||||
}
|
||||
|
||||
/**
|
||||
* 新增知识库附件
|
||||
*
|
||||
* @param bo 知识库附件
|
||||
* @return 是否新增成功
|
||||
*/
|
||||
@Override
|
||||
public Boolean insertByBo(KnowledgeAttachBo bo) {
|
||||
KnowledgeAttach add = MapstructUtils.convert(bo, KnowledgeAttach.class);
|
||||
validEntityBeforeSave(add);
|
||||
boolean flag = baseMapper.insert(add) > 0;
|
||||
if (flag) {
|
||||
bo.setId(add.getId());
|
||||
@@ -124,98 +118,109 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
|
||||
return flag;
|
||||
}
|
||||
|
||||
/**
|
||||
* 修改知识库附件
|
||||
*
|
||||
* @param bo 知识库附件
|
||||
* @return 是否修改成功
|
||||
*/
|
||||
@Override
|
||||
public Boolean updateByBo(KnowledgeAttachBo bo) {
|
||||
KnowledgeAttach update = MapstructUtils.convert(bo, KnowledgeAttach.class);
|
||||
validEntityBeforeSave(update);
|
||||
return baseMapper.updateById(update) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存前的数据校验
|
||||
*/
|
||||
private void validEntityBeforeSave(KnowledgeAttach entity){
|
||||
//TODO 做一些数据校验,如唯一约束
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验并批量删除知识库附件信息
|
||||
*
|
||||
* @param ids 待删除的主键集合
|
||||
* @param isValid 是否进行有效性校验
|
||||
* @return 是否删除成功
|
||||
*/
|
||||
@Override
|
||||
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
|
||||
if(isValid){
|
||||
//TODO 做一些业务上的校验,判断是否需要校验
|
||||
}
|
||||
return baseMapper.deleteByIds(ids) > 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void upload(KnowledgeInfoUploadBo bo) {
|
||||
MultipartFile file = bo.getFile();
|
||||
// 保存文件信息
|
||||
OssDTO ossDTO = ossService.uploadFile(file);
|
||||
Long knowledgeId = bo.getKnowledgeId();
|
||||
List<String> chunkList = new ArrayList<>();
|
||||
|
||||
KnowledgeAttach knowledgeAttach = new KnowledgeAttach();
|
||||
knowledgeAttach.setKnowledgeId(bo.getKnowledgeId());
|
||||
String docId = RandomUtil.randomString(10);
|
||||
knowledgeAttach.setOssId(ossDTO.getOssId());
|
||||
knowledgeAttach.setDocId(docId);
|
||||
knowledgeAttach.setDocId(RandomUtil.randomString(10));
|
||||
knowledgeAttach.setName(ossDTO.getOriginalName());
|
||||
knowledgeAttach.setType(ossDTO.getFileSuffix());
|
||||
String content = "";
|
||||
ResourceLoader resourceLoader = resourceLoaderFactory.getLoaderByFileType(knowledgeAttach.getType());
|
||||
// 文档分段入库
|
||||
List<String> fids = new ArrayList<>();
|
||||
knowledgeAttach.setStatus(KnowledgeAttachStatus.WAITING.getCode()); // 待解析
|
||||
|
||||
baseMapper.insert(knowledgeAttach);
|
||||
|
||||
if (Boolean.TRUE.equals(bo.getAutoParse())) {
|
||||
// 通过 SpringUtils 获取代理对象,确保 @Async 生效
|
||||
SpringUtils.getBean(IKnowledgeAttachService.class).parse(knowledgeAttach.getId());
|
||||
}
|
||||
}
|
||||
|
||||
@Async("knowledgeParseExecutor")
|
||||
@Override
|
||||
public void parse(Long id) {
|
||||
KnowledgeAttach attach = baseMapper.selectById(id);
|
||||
if (attach == null || (!KnowledgeAttachStatus.WAITING.getCode().equals(attach.getStatus()) && !KnowledgeAttachStatus.FAILED.getCode().equals(attach.getStatus()))) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
content = resourceLoader.getContent(file.getInputStream());
|
||||
chunkList = resourceLoader.getChunkList(content, String.valueOf(knowledgeId));
|
||||
attach.setStatus(KnowledgeAttachStatus.PARSING.getCode()); // 解析中
|
||||
baseMapper.updateById(attach);
|
||||
|
||||
log.info("开始解析知识库文档... id: {}, docId: {}", id, attach.getDocId());
|
||||
|
||||
Long knowledgeId = attach.getKnowledgeId();
|
||||
String docId = attach.getDocId();
|
||||
|
||||
// 获取文件信息并下载
|
||||
List<OssDTO> ossDTOs = ossService.selectByIds(String.valueOf(attach.getOssId()));
|
||||
if (ossDTOs == null || ossDTOs.isEmpty()) {
|
||||
throw new RuntimeException("未找到对应的 OSS 文件信息");
|
||||
}
|
||||
OssDTO ossDTO = ossDTOs.get(0);
|
||||
String content;
|
||||
ResourceLoader resourceLoader = resourceLoaderFactory.getLoaderByFileType(attach.getType());
|
||||
try (InputStream inputStream = new URL(ossDTO.getUrl()).openStream()) {
|
||||
content = resourceLoader.getContent(inputStream);
|
||||
}
|
||||
List<String> chunkList = resourceLoader.getChunkList(content, String.valueOf(knowledgeId));
|
||||
|
||||
List<String> fids = new ArrayList<>();
|
||||
List<KnowledgeFragment> knowledgeFragmentList = new ArrayList<>();
|
||||
if (CollUtil.isNotEmpty(chunkList)) {
|
||||
for (int i = 0; i < chunkList.size(); i++) {
|
||||
// 生成知识片段ID
|
||||
String fid = RandomUtil.randomString(10);
|
||||
fids.add(fid);
|
||||
KnowledgeFragment knowledgeFragment = new KnowledgeFragment();
|
||||
knowledgeFragment.setKnowledgeId(knowledgeId);
|
||||
knowledgeFragment.setDocId(docId);
|
||||
knowledgeFragment.setIdx(i);
|
||||
knowledgeFragment.setContent(chunkList.get(i));
|
||||
knowledgeFragment.setCreateTime(new Date());
|
||||
knowledgeFragmentList.add(knowledgeFragment);
|
||||
}
|
||||
knowledgeFragmentMapper.delete(Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
|
||||
knowledgeFragmentMapper.insertBatch(knowledgeFragmentList);
|
||||
log.info("文档切片并入库完成,共计 {} 个片段。id: {}", chunkList.size(), id);
|
||||
}
|
||||
knowledgeFragmentMapper.insertBatch(knowledgeFragmentList);
|
||||
} catch (IOException e) {
|
||||
log.error("保存知识库信息失败!{}", e.getMessage());
|
||||
|
||||
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
|
||||
ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel());
|
||||
|
||||
StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo();
|
||||
storeEmbeddingBo.setKid(String.valueOf(knowledgeId));
|
||||
storeEmbeddingBo.setDocId(docId);
|
||||
storeEmbeddingBo.setFids(fids);
|
||||
storeEmbeddingBo.setChunkList(chunkList);
|
||||
storeEmbeddingBo.setVectorStoreName(knowledgeInfoVo.getVectorModel());
|
||||
storeEmbeddingBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModel());
|
||||
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
|
||||
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
|
||||
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
|
||||
|
||||
attach.setStatus(KnowledgeAttachStatus.COMPLETED.getCode()); // 已完成
|
||||
baseMapper.updateById(attach);
|
||||
log.info("知识库文档解析、向量化并入库成功!id: {}", id);
|
||||
} catch (Exception e) {
|
||||
log.error("解析文档失败!id: {}, error: {}", id, e.getMessage(), e);
|
||||
attach.setStatus(KnowledgeAttachStatus.FAILED.getCode()); // 失败
|
||||
attach.setRemark(StringUtils.substring(e.getMessage(), 0, 255)); // 保存错误原因,截取防止溢出
|
||||
baseMapper.updateById(attach);
|
||||
}
|
||||
baseMapper.insert(knowledgeAttach);
|
||||
|
||||
// 查询知识库信息
|
||||
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
|
||||
|
||||
// 查询向量模信息
|
||||
ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel());
|
||||
|
||||
StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo();
|
||||
storeEmbeddingBo.setKid(String.valueOf(knowledgeId));
|
||||
storeEmbeddingBo.setDocId(docId);
|
||||
storeEmbeddingBo.setFids(fids);
|
||||
storeEmbeddingBo.setChunkList(chunkList);
|
||||
storeEmbeddingBo.setVectorStoreName(knowledgeInfoVo.getVectorModel());
|
||||
storeEmbeddingBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModel());
|
||||
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
|
||||
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
|
||||
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,24 +1,29 @@
|
||||
package org.ruoyi.service.knowledge.impl;
|
||||
|
||||
import org.ruoyi.common.core.utils.MapstructUtils;
|
||||
import org.ruoyi.common.core.utils.StringUtils;
|
||||
import org.ruoyi.common.mybatis.core.page.TableDataInfo;
|
||||
import org.ruoyi.common.mybatis.core.page.PageQuery;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.common.chat.service.chat.IChatModelService;
|
||||
import org.ruoyi.common.core.utils.MapstructUtils;
|
||||
import org.ruoyi.common.core.utils.StringUtils;
|
||||
import org.ruoyi.common.mybatis.core.page.PageQuery;
|
||||
import org.ruoyi.common.mybatis.core.page.TableDataInfo;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeFragmentBo;
|
||||
import org.ruoyi.domain.bo.vector.QueryVectorBo;
|
||||
import org.ruoyi.domain.entity.knowledge.KnowledgeFragment;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeFragmentVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo;
|
||||
import org.ruoyi.mapper.knowledge.KnowledgeFragmentMapper;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeFragmentService;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
|
||||
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Collection;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 知识片段Service业务层处理
|
||||
@@ -32,6 +37,9 @@ import java.util.Collection;
|
||||
public class KnowledgeFragmentServiceImpl implements IKnowledgeFragmentService {
|
||||
|
||||
private final KnowledgeFragmentMapper baseMapper;
|
||||
private final IKnowledgeInfoService knowledgeInfoService;
|
||||
private final IChatModelService chatModelService;
|
||||
private final KnowledgeRetrievalService knowledgeRetrievalService;
|
||||
|
||||
/**
|
||||
* 查询知识片段
|
||||
@@ -71,7 +79,6 @@ public class KnowledgeFragmentServiceImpl implements IKnowledgeFragmentService {
|
||||
}
|
||||
|
||||
private LambdaQueryWrapper<KnowledgeFragment> buildQueryWrapper(KnowledgeFragmentBo bo) {
|
||||
Map<String, Object> params = bo.getParams();
|
||||
LambdaQueryWrapper<KnowledgeFragment> lqw = Wrappers.lambdaQuery();
|
||||
lqw.orderByAsc(KnowledgeFragment::getId);
|
||||
lqw.eq(bo.getDocId() != null, KnowledgeFragment::getDocId, bo.getDocId());
|
||||
@@ -131,4 +138,50 @@ public class KnowledgeFragmentServiceImpl implements IKnowledgeFragmentService {
|
||||
}
|
||||
return baseMapper.deleteByIds(ids) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检索测试核心实现 - 委托给统一的 KnowledgeRetrievalService
|
||||
*/
|
||||
@Override
|
||||
public List<KnowledgeRetrievalVo> retrieval(KnowledgeFragmentBo bo) {
|
||||
if (bo.getKnowledgeId() == null || StringUtils.isBlank(bo.getQuery())) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
// 1. 获取知识库及模型配置(为了获取 API Key/Host 等模型参数)
|
||||
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(bo.getKnowledgeId());
|
||||
if (knowledgeInfoVo == null) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
ChatModelVo chatModel = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel());
|
||||
if (chatModel == null) {
|
||||
log.warn("未找到对应的向量模型配置: {}", knowledgeInfoVo.getEmbeddingModel());
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
// 2. 构造通用的参数对象
|
||||
QueryVectorBo queryVectorBo = new QueryVectorBo();
|
||||
queryVectorBo.setQuery(bo.getQuery());
|
||||
queryVectorBo.setKid(String.valueOf(bo.getKnowledgeId()));
|
||||
queryVectorBo.setApiKey(chatModel.getApiKey());
|
||||
queryVectorBo.setBaseUrl(chatModel.getApiHost());
|
||||
queryVectorBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModel());
|
||||
queryVectorBo.setVectorModelName(knowledgeInfoVo.getVectorModel());
|
||||
|
||||
// 使用前端传入的实时测试参数,若无则使用知识库默认参数
|
||||
queryVectorBo.setMaxResults(bo.getTopK() != null ? bo.getTopK() : knowledgeInfoVo.getRetrieveLimit());
|
||||
queryVectorBo.setSimilarityThreshold(bo.getThreshold() != null ? bo.getThreshold() : knowledgeInfoVo.getSimilarityThreshold());
|
||||
|
||||
queryVectorBo.setEnableHybrid(bo.getEnableHybrid() != null ? bo.getEnableHybrid() : Objects.equals(knowledgeInfoVo.getEnableHybrid(), 1));
|
||||
queryVectorBo.setHybridAlpha(bo.getHybridAlpha() != null ? bo.getHybridAlpha() : knowledgeInfoVo.getHybridAlpha());
|
||||
|
||||
queryVectorBo.setEnableRerank(bo.getEnableRerank() != null ? bo.getEnableRerank() : Objects.equals(knowledgeInfoVo.getEnableRerank(), 1));
|
||||
queryVectorBo.setRerankModelName(StringUtils.isNotBlank(bo.getRerankModel()) ? bo.getRerankModel() : knowledgeInfoVo.getRerankModel());
|
||||
queryVectorBo.setRerankTopN(bo.getTopK() != null ? bo.getTopK() : knowledgeInfoVo.getRerankTopN());
|
||||
queryVectorBo.setRerankScoreThreshold(bo.getThreshold() != null ? bo.getThreshold() : knowledgeInfoVo.getRerankScoreThreshold());
|
||||
|
||||
// 3. 执行统一检索
|
||||
return knowledgeRetrievalService.retrieve(queryVectorBo);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.domain.bo.knowledge.KnowledgeInfoBo;
|
||||
import org.ruoyi.domain.entity.knowledge.KnowledgeInfo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
|
||||
import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper;
|
||||
import org.ruoyi.mapper.knowledge.KnowledgeInfoMapper;
|
||||
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -33,6 +34,8 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
|
||||
|
||||
private final KnowledgeInfoMapper baseMapper;
|
||||
|
||||
private final KnowledgeAttachMapper knowledgeAttachMapper;
|
||||
|
||||
/**
|
||||
* 查询知识库
|
||||
*
|
||||
@@ -55,6 +58,8 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
|
||||
public TableDataInfo<KnowledgeInfoVo> queryPageList(KnowledgeInfoBo bo, PageQuery pageQuery) {
|
||||
LambdaQueryWrapper<KnowledgeInfo> lqw = buildQueryWrapper(bo);
|
||||
Page<KnowledgeInfoVo> result = baseMapper.selectVoPage(pageQuery.build(), lqw);
|
||||
// 批量填充文档数
|
||||
fillDocumentCount(result.getRecords());
|
||||
return TableDataInfo.build(result);
|
||||
}
|
||||
|
||||
@@ -87,6 +92,17 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
|
||||
return lqw;
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量填充知识库列表每一条记录的文档数(documentCount)
|
||||
*/
|
||||
private void fillDocumentCount(List<KnowledgeInfoVo> records) {
|
||||
if (records == null || records.isEmpty()) return;
|
||||
for (KnowledgeInfoVo vo : records) {
|
||||
int count = knowledgeAttachMapper.countByKnowledgeId(vo.getId());
|
||||
vo.setDocumentCount(count);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 新增知识库
|
||||
*
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package org.ruoyi.service.knowledge.retriever;
|
||||
|
||||
import dev.langchain4j.data.segment.TextSegment;
|
||||
import dev.langchain4j.rag.content.Content;
|
||||
import dev.langchain4j.rag.content.retriever.ContentRetriever;
|
||||
import dev.langchain4j.rag.query.Query;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.domain.bo.vector.QueryVectorBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
|
||||
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 自定义检索器:适配 LangChain4j ContentRetriever 接口
|
||||
* 桥接统一的 KnowledgeRetrievalService,支持配置化的混合检索、阈值过滤等功能
|
||||
*
|
||||
* @author RobustH
|
||||
*/
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class CustomVectorRetriever implements ContentRetriever {
|
||||
|
||||
private final KnowledgeRetrievalService knowledgeRetrievalService;
|
||||
private final KnowledgeInfoVo knowledgeInfoVo;
|
||||
private final ChatModelVo chatModelVo;
|
||||
|
||||
@Override
|
||||
public List<Content> retrieve(Query query) {
|
||||
log.info("执行自定义检索,关键字: {}", query.text());
|
||||
|
||||
// 构建增强后的查询参数
|
||||
QueryVectorBo queryVectorBo = new QueryVectorBo();
|
||||
queryVectorBo.setQuery(query.text());
|
||||
queryVectorBo.setKid(String.valueOf(knowledgeInfoVo.getId()));
|
||||
queryVectorBo.setApiKey(chatModelVo.getApiKey());
|
||||
queryVectorBo.setBaseUrl(chatModelVo.getApiHost());
|
||||
queryVectorBo.setVectorModelName(knowledgeInfoVo.getVectorModel());
|
||||
queryVectorBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModel());
|
||||
|
||||
// 应用知识库配置参数
|
||||
queryVectorBo.setMaxResults(knowledgeInfoVo.getRetrieveLimit());
|
||||
queryVectorBo.setSimilarityThreshold(knowledgeInfoVo.getSimilarityThreshold());
|
||||
queryVectorBo.setEnableHybrid(Objects.equals(knowledgeInfoVo.getEnableHybrid(), 1));
|
||||
queryVectorBo.setHybridAlpha(knowledgeInfoVo.getHybridAlpha());
|
||||
|
||||
// 设置重排序参数 (如果 retriever 阶段也想做初步重排,可以在此设置)
|
||||
queryVectorBo.setEnableRerank(Objects.equals(knowledgeInfoVo.getEnableRerank(), 1));
|
||||
queryVectorBo.setRerankModelName(knowledgeInfoVo.getRerankModel());
|
||||
queryVectorBo.setRerankTopN(knowledgeInfoVo.getRerankTopN());
|
||||
queryVectorBo.setRerankScoreThreshold(knowledgeInfoVo.getRerankScoreThreshold());
|
||||
|
||||
// 通过统一服务执行检索
|
||||
List<String> nearestList = knowledgeRetrievalService.retrieveTexts(queryVectorBo);
|
||||
|
||||
// 将结果包装为标准的 Content 返回
|
||||
return nearestList.stream()
|
||||
.map(text -> Content.from(TextSegment.from(text)))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package org.ruoyi.service.rerank;
|
||||
|
||||
import dev.langchain4j.data.segment.TextSegment;
|
||||
import dev.langchain4j.model.output.Response;
|
||||
import dev.langchain4j.model.scoring.ScoringModel;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.domain.bo.rerank.RerankRequest;
|
||||
import org.ruoyi.domain.bo.rerank.RerankResult;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 重排序模型服务接口
|
||||
* 继承 langchain4j 的 ScoringModel 接口
|
||||
* 参考设计模式:BaseEmbedModelService
|
||||
*
|
||||
* @author Yzm
|
||||
* @date 2026-04-19
|
||||
*/
|
||||
public interface RerankModelService extends ScoringModel {
|
||||
|
||||
/**
|
||||
* 根据配置信息配置重排序模型
|
||||
*
|
||||
* @param config 包含模型配置信息的 ChatModelVo 对象
|
||||
*/
|
||||
void configure(ChatModelVo config);
|
||||
|
||||
/**
|
||||
* 执行重排序(批量文档)
|
||||
* 这是业务层使用的便捷方法
|
||||
*
|
||||
* @param rerankRequest 重排序请求,包含查询文本和候选文档列表
|
||||
* @return 重排序结果,包含排序后的文档和相关性分数
|
||||
*/
|
||||
RerankResult rerank(RerankRequest rerankRequest);
|
||||
|
||||
/**
|
||||
* 实现 ScoringModel 接口的 scoreAll 方法
|
||||
* 将 ScoringModel 的调用转换为重排序调用
|
||||
*/
|
||||
@Override
|
||||
default Response<List<Double>> scoreAll(List<TextSegment> segments, String query) {
|
||||
// 将 TextSegment 转换为文档字符串列表
|
||||
List<String> documents = segments.stream()
|
||||
.map(TextSegment::text)
|
||||
.toList();
|
||||
|
||||
RerankRequest request = RerankRequest.builder()
|
||||
.query(query)
|
||||
.documents(documents)
|
||||
.topN(documents.size())
|
||||
.returnDocuments(false)
|
||||
.build();
|
||||
|
||||
RerankResult result = rerank(request);
|
||||
|
||||
// 提取分数列表,按原始顺序排列
|
||||
List<Double> scores = new java.util.ArrayList<>(
|
||||
java.util.Collections.nCopies(documents.size(), 0.0));
|
||||
|
||||
for (RerankResult.RerankDocument doc : result.getDocuments()) {
|
||||
if (doc.getIndex() != null && doc.getIndex() < documents.size()) {
|
||||
scores.set(doc.getIndex(), doc.getRelevanceScore());
|
||||
}
|
||||
}
|
||||
|
||||
return Response.from(scores);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package org.ruoyi.service.rerank.impl;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.*;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.domain.bo.rerank.RerankRequest;
|
||||
import org.ruoyi.domain.bo.rerank.RerankResult;
|
||||
import org.ruoyi.domain.dto.request.AliBaiLianRerankRequest;
|
||||
import org.ruoyi.domain.dto.response.AliBaiLianRerankResponse;
|
||||
import org.ruoyi.service.rerank.RerankModelService;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 阿里百炼重排序模型实现
|
||||
* 参考设计模式:AliBaiLianMultiEmbeddingProvider
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-20
|
||||
*/
|
||||
@Slf4j
|
||||
@Component("qianwenRerank")
|
||||
public class AliBaiLianRerankModelService implements RerankModelService {
|
||||
|
||||
private final OkHttpClient okHttpClient;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private ChatModelVo chatModelVo;
|
||||
|
||||
public AliBaiLianRerankModelService() {
|
||||
this.okHttpClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(30, TimeUnit.SECONDS)
|
||||
.readTimeout(60, TimeUnit.SECONDS)
|
||||
.writeTimeout(30, TimeUnit.SECONDS)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(ChatModelVo config) {
|
||||
this.chatModelVo = config;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RerankResult rerank(RerankRequest rerankRequest) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
try {
|
||||
// 构建请求
|
||||
AliBaiLianRerankRequest request = buildRequest(rerankRequest);
|
||||
AliBaiLianRerankResponse response = executeRequest(request);
|
||||
|
||||
return response.toRerankResult(
|
||||
rerankRequest.getDocuments().size(),
|
||||
System.currentTimeMillis() - startTime
|
||||
);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("阿里百炼重排序失败: {}", e.getMessage(), e);
|
||||
throw new RuntimeException("重排序服务调用失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建请求对象
|
||||
*/
|
||||
private AliBaiLianRerankRequest buildRequest(RerankRequest rerankRequest) {
|
||||
return AliBaiLianRerankRequest.create(
|
||||
chatModelVo.getModelName(),
|
||||
rerankRequest.getQuery(),
|
||||
rerankRequest.getDocuments(),
|
||||
rerankRequest.getTopN(),
|
||||
rerankRequest.getReturnDocuments()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行HTTP请求并解析响应
|
||||
*/
|
||||
private AliBaiLianRerankResponse executeRequest(AliBaiLianRerankRequest request) throws IOException {
|
||||
String jsonBody = request.toJson();
|
||||
RequestBody body = RequestBody.create(jsonBody, MediaType.get("application/json"));
|
||||
|
||||
// 阿里百炼重排序 OpenAI兼容端点
|
||||
String url = chatModelVo.getApiHost() + "/compatible-api/v1/reranks";
|
||||
Request httpRequest = new Request.Builder()
|
||||
.url(url)
|
||||
.addHeader("Authorization", "Bearer " + chatModelVo.getApiKey())
|
||||
.addHeader("Content-Type", "application/json")
|
||||
.post(body)
|
||||
.build();
|
||||
|
||||
try (Response response = okHttpClient.newCall(httpRequest).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
String err = response.body() != null ? response.body().string() : "无错误信息";
|
||||
throw new IllegalArgumentException("阿里百炼API调用失败: " + response.code() + " - " + err);
|
||||
}
|
||||
|
||||
ResponseBody responseBody = response.body();
|
||||
if (responseBody == null) {
|
||||
throw new IllegalArgumentException("响应体为空");
|
||||
}
|
||||
|
||||
return parseResponse(responseBody.string());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析响应
|
||||
*/
|
||||
private AliBaiLianRerankResponse parseResponse(String responseBody) throws IOException {
|
||||
return objectMapper.readValue(responseBody, AliBaiLianRerankResponse.class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package org.ruoyi.service.rerank.impl;
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.*;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.domain.bo.rerank.RerankRequest;
|
||||
import org.ruoyi.domain.bo.rerank.RerankResult;
|
||||
import org.ruoyi.service.rerank.RerankModelService;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 硅基流动重排序模型实现
|
||||
* 适配硅基流动的 /v1/rerank 接口
|
||||
*
|
||||
* @author RobustH
|
||||
* @date 2026-04-21
|
||||
*/
|
||||
@Slf4j
|
||||
@Component("siliconflowRerank")
|
||||
public class SiliconFlowRerankModelService implements RerankModelService {
|
||||
|
||||
private static final String DEFAULT_BASE_URL = "https://api.siliconflow.cn/v1/rerank";
|
||||
|
||||
private final OkHttpClient okHttpClient;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper()
|
||||
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
private ChatModelVo chatModelVo;
|
||||
|
||||
public SiliconFlowRerankModelService() {
|
||||
this.okHttpClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(30, TimeUnit.SECONDS)
|
||||
.readTimeout(60, TimeUnit.SECONDS)
|
||||
.writeTimeout(30, TimeUnit.SECONDS)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(ChatModelVo config) {
|
||||
this.chatModelVo = config;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RerankResult rerank(RerankRequest rerankRequest) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
try {
|
||||
String url = buildUrl();
|
||||
String requestJson = buildRequestJson(rerankRequest);
|
||||
|
||||
RequestBody body = RequestBody.create(requestJson, MediaType.get("application/json"));
|
||||
Request httpRequest = new Request.Builder()
|
||||
.url(url)
|
||||
.addHeader("Authorization", "Bearer " + chatModelVo.getApiKey())
|
||||
.addHeader("Content-Type", "application/json")
|
||||
.post(body)
|
||||
.build();
|
||||
|
||||
log.info("硅基流动重排序请求: model={}, url={}", chatModelVo.getModelName(), url);
|
||||
|
||||
try (Response response = okHttpClient.newCall(httpRequest).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
String err = response.body() != null ? response.body().string() : "无错误信息";
|
||||
throw new IllegalArgumentException("硅基流动 Rerank API 调用失败: " + response.code() + " - " + err);
|
||||
}
|
||||
|
||||
ResponseBody responseBody = response.body();
|
||||
if (responseBody == null) {
|
||||
throw new IllegalArgumentException("响应体为空");
|
||||
}
|
||||
|
||||
SiliconFlowRerankResponse rerankResponse = objectMapper.readValue(
|
||||
responseBody.string(), SiliconFlowRerankResponse.class);
|
||||
|
||||
return buildRerankResult(rerankResponse, rerankRequest.getDocuments(),
|
||||
System.currentTimeMillis() - startTime);
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("硅基流动重排序失败: {}", e.getMessage(), e);
|
||||
throw new RuntimeException("重排序服务调用失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建请求 URL,鲁棒处理 API Host 末尾路径
|
||||
*/
|
||||
private String buildUrl() {
|
||||
String apiHost = chatModelVo.getApiHost();
|
||||
if (apiHost == null || apiHost.isBlank()) {
|
||||
return DEFAULT_BASE_URL;
|
||||
}
|
||||
if (apiHost.endsWith("/rerank")) {
|
||||
return apiHost;
|
||||
}
|
||||
if (apiHost.endsWith("/v1")) {
|
||||
return apiHost + "/rerank";
|
||||
}
|
||||
return apiHost.endsWith("/") ? apiHost + "rerank" : apiHost + "/rerank";
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建请求体 JSON
|
||||
*/
|
||||
private String buildRequestJson(RerankRequest rerankRequest) throws IOException {
|
||||
SiliconFlowRerankRequest request = new SiliconFlowRerankRequest();
|
||||
request.setModel(chatModelVo.getModelName());
|
||||
request.setQuery(rerankRequest.getQuery());
|
||||
request.setDocuments(rerankRequest.getDocuments());
|
||||
request.setTop_n(rerankRequest.getTopN() != null ? rerankRequest.getTopN() : rerankRequest.getDocuments().size());
|
||||
request.setReturn_documents(rerankRequest.getReturnDocuments() != null ? rerankRequest.getReturnDocuments() : false);
|
||||
return objectMapper.writeValueAsString(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建标准 RerankResult
|
||||
*/
|
||||
private RerankResult buildRerankResult(SiliconFlowRerankResponse response,
|
||||
List<String> originalDocuments, long durationMs) {
|
||||
Double[] scores = new Double[originalDocuments.size()];
|
||||
for (int i = 0; i < scores.length; i++) {
|
||||
scores[i] = 0.0;
|
||||
}
|
||||
|
||||
List<RerankResult.RerankDocument> docs = new ArrayList<>();
|
||||
if (response != null && response.getResults() != null) {
|
||||
response.getResults().forEach(item -> {
|
||||
if (item.getIndex() != null && item.getIndex() < originalDocuments.size()) {
|
||||
scores[item.getIndex()] = item.getRelevance_score();
|
||||
docs.add(RerankResult.RerankDocument.builder()
|
||||
.index(item.getIndex())
|
||||
.relevanceScore(item.getRelevance_score())
|
||||
.document(originalDocuments.get(item.getIndex()))
|
||||
.build());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return RerankResult.builder()
|
||||
.documents(docs)
|
||||
.totalDocuments(originalDocuments.size())
|
||||
.durationMs(durationMs)
|
||||
.build();
|
||||
}
|
||||
|
||||
// ==================== 内部 DTO ====================
|
||||
|
||||
@Data
|
||||
static class SiliconFlowRerankRequest {
|
||||
private String model;
|
||||
private String query;
|
||||
private List<String> documents;
|
||||
private Integer top_n;
|
||||
private Boolean return_documents;
|
||||
}
|
||||
|
||||
@Data
|
||||
static class SiliconFlowRerankResponse {
|
||||
private List<SiliconFlowRerankResultItem> results;
|
||||
}
|
||||
|
||||
@Data
|
||||
static class SiliconFlowRerankResultItem {
|
||||
private Integer index;
|
||||
private Double relevance_score;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package org.ruoyi.service.rerank.impl;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import io.jsonwebtoken.Jwts;
|
||||
import io.jsonwebtoken.security.MacAlgorithm;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.*;
|
||||
import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo;
|
||||
import org.ruoyi.domain.bo.rerank.RerankRequest;
|
||||
import org.ruoyi.domain.bo.rerank.RerankResult;
|
||||
import org.ruoyi.domain.dto.request.ZhipuRerankRequest;
|
||||
import org.ruoyi.domain.dto.response.ZhipuRerankResponse;
|
||||
import org.ruoyi.service.rerank.RerankModelService;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 智谱AI 重排序模型实现
|
||||
* 参考设计模式:AliBaiLianMultiEmbeddingProvider
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-19
|
||||
*/
|
||||
@Slf4j
|
||||
@Component("zhipuRerank")
|
||||
public class ZhiPuRerankModelService implements RerankModelService {
|
||||
|
||||
private final OkHttpClient okHttpClient;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private ChatModelVo chatModelVo;
|
||||
|
||||
public ZhiPuRerankModelService() {
|
||||
this.okHttpClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(30, TimeUnit.SECONDS)
|
||||
.readTimeout(60, TimeUnit.SECONDS)
|
||||
.writeTimeout(30, TimeUnit.SECONDS)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(ChatModelVo config) {
|
||||
this.chatModelVo = config;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RerankResult rerank(RerankRequest rerankRequest) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
try {
|
||||
// 构建请求
|
||||
ZhipuRerankRequest request = buildRequest(rerankRequest);
|
||||
ZhipuRerankResponse response = executeRequest(request);
|
||||
|
||||
return response.toRerankResult(
|
||||
rerankRequest.getDocuments().size(),
|
||||
System.currentTimeMillis() - startTime
|
||||
);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("智谱重排序失败: {}", e.getMessage(), e);
|
||||
throw new RuntimeException("重排序服务调用失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建请求对象
|
||||
*/
|
||||
private ZhipuRerankRequest buildRequest(RerankRequest rerankRequest) {
|
||||
return ZhipuRerankRequest.create(
|
||||
chatModelVo.getModelName(),
|
||||
rerankRequest.getQuery(),
|
||||
rerankRequest.getDocuments(),
|
||||
rerankRequest.getTopN(),
|
||||
rerankRequest.getReturnDocuments()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行HTTP请求并解析响应
|
||||
*/
|
||||
private ZhipuRerankResponse executeRequest(ZhipuRerankRequest request) throws IOException {
|
||||
String jsonBody = request.toJson();
|
||||
RequestBody body = RequestBody.create(jsonBody, MediaType.get("application/json"));
|
||||
|
||||
// 生成智谱认证Token
|
||||
String token = generateToken(chatModelVo.getApiKey());
|
||||
|
||||
// 智谱重排序固定端点路径
|
||||
String url = chatModelVo.getApiHost() + "/api/paas/v4/rerank";
|
||||
Request httpRequest = new Request.Builder()
|
||||
.url(url)
|
||||
.addHeader("Authorization", token)
|
||||
.post(body)
|
||||
.build();
|
||||
|
||||
try (Response response = okHttpClient.newCall(httpRequest).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
String err = response.body() != null ? response.body().string() : "无错误信息";
|
||||
throw new IllegalArgumentException("智谱API调用失败: " + response.code() + " - " + err);
|
||||
}
|
||||
|
||||
ResponseBody responseBody = response.body();
|
||||
if (responseBody == null) {
|
||||
throw new IllegalArgumentException("响应体为空");
|
||||
}
|
||||
|
||||
return parseResponse(responseBody.string());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析响应
|
||||
*/
|
||||
private ZhipuRerankResponse parseResponse(String responseBody) throws IOException {
|
||||
return objectMapper.readValue(responseBody, ZhipuRerankResponse.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成智谱JWT Token
|
||||
*/
|
||||
private String generateToken(String apiKey) {
|
||||
try {
|
||||
String[] apiKeyParts = apiKey.split("\\.");
|
||||
String keyId = apiKeyParts[0];
|
||||
String secret = apiKeyParts[1];
|
||||
|
||||
long expireMillis = 1000L * 60 * 30; // 30分钟
|
||||
java.util.Map<String, Object> payload = new java.util.HashMap<>();
|
||||
payload.put("api_key", keyId);
|
||||
payload.put("exp", System.currentTimeMillis() + expireMillis);
|
||||
payload.put("timestamp", System.currentTimeMillis());
|
||||
|
||||
// 使用反射创建 MacAlgorithm(兼容不同版本的 jjwt)
|
||||
MacAlgorithm macAlgorithm;
|
||||
try {
|
||||
Class<?> c = Class.forName("io.jsonwebtoken.impl.security.DefaultMacAlgorithm");
|
||||
Constructor<?> ctor = c.getDeclaredConstructor(String.class, String.class, int.class);
|
||||
ctor.setAccessible(true);
|
||||
macAlgorithm = (MacAlgorithm) ctor.newInstance("HS256", "HmacSHA256", 128);
|
||||
} catch (Exception e) {
|
||||
macAlgorithm = Jwts.SIG.HS256;
|
||||
}
|
||||
|
||||
String token = Jwts.builder()
|
||||
.header()
|
||||
.add("alg", "HS256")
|
||||
.add("sign_type", "SIGN")
|
||||
.and()
|
||||
.content(objectMapper.writeValueAsString(payload))
|
||||
.signWith(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256"), macAlgorithm)
|
||||
.compact();
|
||||
|
||||
return "Bearer " + token;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("生成智谱Token失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package org.ruoyi.service.retrieval;
|
||||
|
||||
import org.ruoyi.domain.bo.vector.QueryVectorBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 知识库检索服务接口
|
||||
* 整合粗召回(向量检索/关键词检索)和重排序流程
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-19
|
||||
*/
|
||||
public interface KnowledgeRetrievalService {
|
||||
|
||||
/**
|
||||
* 执行知识库检索,返回文本内容
|
||||
* 流程:向量粗召回 -> 重排序(可选) -> 返回结果
|
||||
*
|
||||
* @param queryVectorBo 查询参数
|
||||
* @return 文本内容列表
|
||||
*/
|
||||
List<String> retrieveTexts(QueryVectorBo queryVectorBo);
|
||||
|
||||
/**
|
||||
* 执行知识库检索,返回详细结果对象(包含分数、文档ID等)
|
||||
* 支持混合检索和重排序
|
||||
*
|
||||
* @param queryVectorBo 查询参数
|
||||
* @return 检索结果列表
|
||||
*/
|
||||
List<KnowledgeRetrievalVo> retrieve(QueryVectorBo queryVectorBo);
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package org.ruoyi.service.retrieval.impl;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ruoyi.common.core.utils.StringUtils;
|
||||
import org.ruoyi.domain.bo.rerank.RerankRequest;
|
||||
import org.ruoyi.domain.bo.rerank.RerankResult;
|
||||
import org.ruoyi.domain.bo.vector.QueryVectorBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeFragmentVo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo;
|
||||
import org.ruoyi.factory.RerankModelFactory;
|
||||
import org.ruoyi.mapper.knowledge.KnowledgeFragmentMapper;
|
||||
import org.ruoyi.service.rerank.RerankModelService;
|
||||
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
|
||||
import org.ruoyi.service.vector.VectorStoreService;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 知识库检索服务实现
|
||||
* 整合粗召回(向量检索/关键词检索)、RRF融合和重排序流程
|
||||
*
|
||||
* @author yang
|
||||
* @date 2026-04-19
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class KnowledgeRetrievalServiceImpl implements KnowledgeRetrievalService {
|
||||
|
||||
private final VectorStoreService vectorStoreService;
|
||||
private final RerankModelFactory rerankModelFactory;
|
||||
private final KnowledgeFragmentMapper fragmentMapper;
|
||||
|
||||
/**
|
||||
* 粗召回默认扩大倍数
|
||||
* 如果启用重排序,粗召回会获取更多结果供重排序筛选
|
||||
*/
|
||||
private static final int RERANK_EXPANSION_FACTOR = 3;
|
||||
|
||||
@Override
|
||||
public List<String> retrieveTexts(QueryVectorBo queryVectorBo) {
|
||||
List<KnowledgeRetrievalVo> results = retrieve(queryVectorBo);
|
||||
return results.stream()
|
||||
.map(KnowledgeRetrievalVo::getContent)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KnowledgeRetrievalVo> retrieve(QueryVectorBo queryVectorBo) {
|
||||
log.info("开始知识库检索, kid={}, query={}", queryVectorBo.getKid(), queryVectorBo.getQuery());
|
||||
|
||||
// 1. 粗召回阶段 (向量检索 + 关键词搜索)
|
||||
List<KnowledgeRetrievalVo> coarseResults = performCoarseRetrieval(queryVectorBo);
|
||||
log.debug("粗召回返回 {} 条结果", coarseResults.size());
|
||||
|
||||
if (coarseResults.isEmpty()) {
|
||||
return coarseResults;
|
||||
}
|
||||
|
||||
// 2. 初始化原始索引
|
||||
for (int i = 0; i < coarseResults.size(); i++) {
|
||||
coarseResults.get(i).setOriginalIndex(i);
|
||||
}
|
||||
|
||||
// 3. 重排序阶段 (可选)
|
||||
List<KnowledgeRetrievalVo> finalResults = coarseResults;
|
||||
if (Boolean.TRUE.equals(queryVectorBo.getEnableRerank()) &&
|
||||
StringUtils.isNotBlank(queryVectorBo.getRerankModelName())) {
|
||||
finalResults = performRerank(queryVectorBo, coarseResults);
|
||||
}
|
||||
|
||||
// 4. 应用分值阈值过滤 (重排分值或 RRF 分值)
|
||||
double threshold = queryVectorBo.getRerankScoreThreshold() != null ?
|
||||
queryVectorBo.getRerankScoreThreshold() : 0.0;
|
||||
|
||||
return finalResults.stream()
|
||||
.filter(res -> res.getScore() >= threshold)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* 粗召回阶段:根据配置执行向量搜索或混合搜索
|
||||
*/
|
||||
private List<KnowledgeRetrievalVo> performCoarseRetrieval(QueryVectorBo queryVectorBo) {
|
||||
// 如果启用重排序,适当扩大召回数量
|
||||
int originalMaxResults = queryVectorBo.getMaxResults() != null ? queryVectorBo.getMaxResults() : 10;
|
||||
int targetMaxResults = originalMaxResults;
|
||||
if (Boolean.TRUE.equals(queryVectorBo.getEnableRerank()) &&
|
||||
StringUtils.isNotBlank(queryVectorBo.getRerankModelName())) {
|
||||
targetMaxResults = originalMaxResults * RERANK_EXPANSION_FACTOR;
|
||||
}
|
||||
|
||||
// 如果未启用混合检索,直接走向量搜索
|
||||
if (!Boolean.TRUE.equals(queryVectorBo.getEnableHybrid())) {
|
||||
QueryVectorBo vectorQuery = copyOf(queryVectorBo, targetMaxResults);
|
||||
List<KnowledgeRetrievalVo> results = vectorStoreService.search(vectorQuery);
|
||||
|
||||
// 应用基础相似度阈值过滤(如果有)
|
||||
if (queryVectorBo.getSimilarityThreshold() != null) {
|
||||
results = results.stream()
|
||||
.filter(r -> r.getScore() >= queryVectorBo.getSimilarityThreshold())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// 混合检索逻辑
|
||||
log.info("执行混合检索: kid={}, query={}", queryVectorBo.getKid(), queryVectorBo.getQuery());
|
||||
try {
|
||||
// A. 并行执行向量搜索
|
||||
int finalTargetMaxResults = targetMaxResults;
|
||||
CompletableFuture<List<KnowledgeRetrievalVo>> vectorFuture = CompletableFuture.supplyAsync(() -> {
|
||||
QueryVectorBo vectorQuery = copyOf(queryVectorBo, finalTargetMaxResults);
|
||||
List<KnowledgeRetrievalVo> results = vectorStoreService.search(vectorQuery);
|
||||
// 向量层初步过滤
|
||||
if (queryVectorBo.getSimilarityThreshold() != null) {
|
||||
return results.stream()
|
||||
.filter(r -> r.getScore() >= queryVectorBo.getSimilarityThreshold())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
return results;
|
||||
});
|
||||
|
||||
// B. 并行执行关键词搜索 (MySQL Fulltext)
|
||||
CompletableFuture<List<KnowledgeRetrievalVo>> keywordFuture = CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
Long kid = Long.valueOf(queryVectorBo.getKid());
|
||||
List<KnowledgeFragmentVo> fragments = fragmentMapper.searchByKeyword(kid, queryVectorBo.getQuery(), finalTargetMaxResults);
|
||||
return fragments.stream().map(f -> {
|
||||
KnowledgeRetrievalVo vo = new KnowledgeRetrievalVo();
|
||||
vo.setId(f.getId().toString());
|
||||
vo.setContent(f.getContent());
|
||||
vo.setDocId(f.getDocId());
|
||||
vo.setIdx(f.getIdx());
|
||||
vo.setKnowledgeId(f.getKnowledgeId());
|
||||
vo.setScore(10.0); // RRF 初始占位分
|
||||
return vo;
|
||||
}).collect(Collectors.toList());
|
||||
} catch (Exception e) {
|
||||
log.error("关键词检索失败: {}", e.getMessage());
|
||||
return new ArrayList<>();
|
||||
}
|
||||
});
|
||||
|
||||
List<KnowledgeRetrievalVo> vectorResults = vectorFuture.get();
|
||||
List<KnowledgeRetrievalVo> keywordResults = keywordFuture.get();
|
||||
|
||||
// C. RRF 融合
|
||||
double alpha = queryVectorBo.getHybridAlpha() != null ? queryVectorBo.getHybridAlpha() : 0.5;
|
||||
return calculateRRF(vectorResults, keywordResults, alpha);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("混合检索执行失败,回退到纯向量检索: {}", e.getMessage(), e);
|
||||
return vectorStoreService.search(copyOf(queryVectorBo, targetMaxResults));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 重排序阶段
|
||||
*/
|
||||
private List<KnowledgeRetrievalVo> performRerank(QueryVectorBo queryVectorBo, List<KnowledgeRetrievalVo> coarseResults) {
|
||||
try {
|
||||
RerankModelService rerankModel = rerankModelFactory.createModel(queryVectorBo.getRerankModelName());
|
||||
|
||||
List<String> contents = coarseResults.stream()
|
||||
.map(KnowledgeRetrievalVo::getContent)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// topN 默认为 maxResults
|
||||
int topN = queryVectorBo.getRerankTopN() != null ? queryVectorBo.getRerankTopN() : queryVectorBo.getMaxResults();
|
||||
|
||||
RerankRequest rerankRequest = RerankRequest.builder()
|
||||
.query(queryVectorBo.getQuery())
|
||||
.documents(contents)
|
||||
.topN(topN)
|
||||
.build();
|
||||
|
||||
RerankResult rerankResult = rerankModel.rerank(rerankRequest);
|
||||
|
||||
// 写回分数并记录原始分
|
||||
for (RerankResult.RerankDocument doc : rerankResult.getDocuments()) {
|
||||
if (doc.getIndex() != null && doc.getIndex() < coarseResults.size()) {
|
||||
KnowledgeRetrievalVo vo = coarseResults.get(doc.getIndex());
|
||||
vo.setRawScore(vo.getScore());
|
||||
vo.setScore(doc.getRelevanceScore());
|
||||
}
|
||||
}
|
||||
|
||||
// 按新分排序
|
||||
coarseResults.sort((a, b) -> b.getScore().compareTo(a.getScore()));
|
||||
|
||||
// 截断到 topN
|
||||
return coarseResults.subList(0, Math.min(topN, coarseResults.size()));
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("重排序流程失败: {}", e.getMessage());
|
||||
int limit = queryVectorBo.getMaxResults() != null ? queryVectorBo.getMaxResults() : 10;
|
||||
return coarseResults.subList(0, Math.min(limit, coarseResults.size()));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* RRF (Reciprocal Rank Fusion) 融合计算
|
||||
*/
|
||||
private List<KnowledgeRetrievalVo> calculateRRF(List<KnowledgeRetrievalVo> vectorList, List<KnowledgeRetrievalVo> keywordList, double alpha) {
|
||||
Map<String, KnowledgeRetrievalVo> allMap = new LinkedHashMap<>();
|
||||
Map<String, Double> vectorScores = new HashMap<>();
|
||||
Map<String, Double> keywordScores = new HashMap<>();
|
||||
|
||||
int k = 60; // RRF 常数
|
||||
|
||||
for (int i = 0; i < vectorList.size(); i++) {
|
||||
KnowledgeRetrievalVo vo = vectorList.get(i);
|
||||
allMap.put(vo.getId(), vo);
|
||||
vectorScores.put(vo.getId(), 1.0 / (k + i + 1));
|
||||
}
|
||||
|
||||
for (int i = 0; i < keywordList.size(); i++) {
|
||||
KnowledgeRetrievalVo vo = keywordList.get(i);
|
||||
if (!allMap.containsKey(vo.getId())) {
|
||||
allMap.put(vo.getId(), vo);
|
||||
}
|
||||
keywordScores.put(vo.getId(), 1.0 / (k + i + 1));
|
||||
}
|
||||
|
||||
List<KnowledgeRetrievalVo> fusedResults = new ArrayList<>();
|
||||
for (Map.Entry<String, KnowledgeRetrievalVo> entry : allMap.entrySet()) {
|
||||
String id = entry.getKey();
|
||||
double finalScore = (1 - alpha) * vectorScores.getOrDefault(id, 0.0) +
|
||||
alpha * keywordScores.getOrDefault(id, 0.0);
|
||||
|
||||
KnowledgeRetrievalVo vo = entry.getValue();
|
||||
vo.setScore(finalScore * 60.0); // 归一化缩放
|
||||
fusedResults.add(vo);
|
||||
}
|
||||
|
||||
fusedResults.sort((a, b) -> b.getScore().compareTo(a.getScore()));
|
||||
return fusedResults;
|
||||
}
|
||||
|
||||
private QueryVectorBo copyOf(QueryVectorBo original, int maxResults) {
|
||||
QueryVectorBo copy = new QueryVectorBo();
|
||||
copy.setQuery(original.getQuery());
|
||||
copy.setKid(original.getKid());
|
||||
copy.setMaxResults(maxResults);
|
||||
copy.setVectorModelName(original.getVectorModelName());
|
||||
copy.setEmbeddingModelName(original.getEmbeddingModelName());
|
||||
copy.setApiKey(original.getApiKey());
|
||||
copy.setBaseUrl(original.getBaseUrl());
|
||||
return copy;
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package org.ruoyi.service.vector;
|
||||
import org.ruoyi.common.core.exception.ServiceException;
|
||||
import org.ruoyi.domain.bo.vector.QueryVectorBo;
|
||||
import org.ruoyi.domain.bo.vector.StoreEmbeddingBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@@ -17,6 +18,11 @@ public interface VectorStoreService {
|
||||
|
||||
List<String> getQueryVector(QueryVectorBo queryVectorBo);
|
||||
|
||||
/**
|
||||
* 带分数及元数据的检索(用于测试检索功能)
|
||||
*/
|
||||
List<KnowledgeRetrievalVo> search(QueryVectorBo queryVectorBo);
|
||||
|
||||
void createSchema(String kid, String embeddingModelName);
|
||||
|
||||
void removeById(String id, String modelName) throws ServiceException;
|
||||
|
||||
@@ -37,6 +37,24 @@ public abstract class AbstractVectorStoreStrategy implements VectorStoreService
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 向量 L2 归一化 (单位化)
|
||||
*/
|
||||
protected static float[] normalize(float[] vector) {
|
||||
if (vector == null) return null;
|
||||
double sum = 0;
|
||||
for (float v : vector) {
|
||||
sum += v * v;
|
||||
}
|
||||
float norm = (float) Math.sqrt(sum);
|
||||
if (norm > 1e-9) {
|
||||
for (int i = 0; i < vector.length; i++) {
|
||||
vector[i] /= norm;
|
||||
}
|
||||
}
|
||||
return vector;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取向量模型
|
||||
*/
|
||||
|
||||
@@ -19,7 +19,11 @@ import org.ruoyi.common.chat.service.chat.IChatModelService;
|
||||
import org.ruoyi.config.VectorStoreProperties;
|
||||
import org.ruoyi.domain.bo.vector.QueryVectorBo;
|
||||
import org.ruoyi.domain.bo.vector.StoreEmbeddingBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo;
|
||||
import org.ruoyi.factory.EmbeddingModelFactory;
|
||||
import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper;
|
||||
import org.ruoyi.domain.entity.knowledge.KnowledgeAttach;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@@ -32,10 +36,14 @@ import java.util.stream.IntStream;
|
||||
@Component
|
||||
public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
||||
|
||||
private final KnowledgeAttachMapper knowledgeAttachMapper;
|
||||
|
||||
public MilvusVectorStoreStrategy(VectorStoreProperties vectorStoreProperties,
|
||||
IChatModelService chatModelService,
|
||||
EmbeddingModelFactory embeddingModelFactory) {
|
||||
EmbeddingModelFactory embeddingModelFactory,
|
||||
KnowledgeAttachMapper knowledgeAttachMapper) {
|
||||
super(vectorStoreProperties, embeddingModelFactory, chatModelService);
|
||||
this.knowledgeAttachMapper = knowledgeAttachMapper;
|
||||
}
|
||||
|
||||
// 缓存不同集合与 autoFlush 配置的 Milvus 连接
|
||||
@@ -51,7 +59,7 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
||||
.collectionName(collectionName)
|
||||
.dimension(dimension)
|
||||
.indexType(IndexType.IVF_FLAT)
|
||||
.metricType(MetricType.L2)
|
||||
.metricType(MetricType.COSINE)
|
||||
.autoFlushOnInsert(autoFlushOnInsert)
|
||||
.idFieldName("id")
|
||||
.textFieldName("text")
|
||||
@@ -104,7 +112,10 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
||||
|
||||
TextSegment textSegment = TextSegment.from(text, metadata);
|
||||
Embedding embedding = embeddingModel.embed(text).content();
|
||||
embeddingStore.add(embedding, textSegment);
|
||||
// 单位化处理
|
||||
float[] vector = embedding.vector();
|
||||
normalize(vector);
|
||||
embeddingStore.add(Embedding.from(vector), textSegment);
|
||||
});
|
||||
long endTime = System.currentTimeMillis();
|
||||
log.info("Milvus向量存储完成消耗时间:{}秒", (endTime - startTime) / 1000);
|
||||
@@ -136,6 +147,55 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
||||
return resultList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KnowledgeRetrievalVo> search(QueryVectorBo queryVectorBo) {
|
||||
int dimension = getModelDimension(queryVectorBo.getEmbeddingModelName());
|
||||
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName());
|
||||
|
||||
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
|
||||
// 查询向量单位化处理
|
||||
float[] queryVector = queryEmbedding.vector();
|
||||
normalize(queryVector);
|
||||
|
||||
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + queryVectorBo.getKid();
|
||||
|
||||
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, dimension, true);
|
||||
|
||||
EmbeddingSearchRequest request = EmbeddingSearchRequest.builder()
|
||||
.queryEmbedding(Embedding.from(queryVector))
|
||||
.maxResults(queryVectorBo.getMaxResults())
|
||||
.build();
|
||||
|
||||
List<EmbeddingMatch<TextSegment>> matches = embeddingStore.search(request).matches();
|
||||
List<KnowledgeRetrievalVo> resultList = new ArrayList<>();
|
||||
|
||||
for (EmbeddingMatch<TextSegment> match : matches) {
|
||||
TextSegment segment = match.embedded();
|
||||
if (segment == null) continue;
|
||||
|
||||
String docId = segment.metadata().getString("docId");
|
||||
String sourceName = "未知来源";
|
||||
if (docId != null) {
|
||||
KnowledgeAttach attach = knowledgeAttachMapper.selectOne(new LambdaQueryWrapper<KnowledgeAttach>()
|
||||
.eq(KnowledgeAttach::getDocId, docId)
|
||||
.last("limit 1"));
|
||||
if (attach != null) {
|
||||
sourceName = attach.getName();
|
||||
}
|
||||
}
|
||||
|
||||
// 提取内容、评分及来源
|
||||
double score = match.score();
|
||||
|
||||
resultList.add(org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo.builder()
|
||||
.content(segment.text())
|
||||
.score(score)
|
||||
.sourceName(sourceName)
|
||||
.build());
|
||||
}
|
||||
return resultList;
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void removeById(String id, String modelName) {
|
||||
|
||||
@@ -24,7 +24,11 @@ import org.ruoyi.common.core.exception.ServiceException;
|
||||
import org.ruoyi.config.VectorStoreProperties;
|
||||
import org.ruoyi.domain.bo.vector.QueryVectorBo;
|
||||
import org.ruoyi.domain.bo.vector.StoreEmbeddingBo;
|
||||
import org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo;
|
||||
import org.ruoyi.factory.EmbeddingModelFactory;
|
||||
import org.ruoyi.domain.entity.knowledge.KnowledgeAttach;
|
||||
import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import static io.qdrant.client.VectorInputFactory.vectorInput;
|
||||
@@ -47,10 +51,14 @@ public class QdrantVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
||||
private static final String METADATA_KID_KEY = "kid";
|
||||
private static final String METADATA_DOC_ID_KEY = "doc_id";
|
||||
|
||||
private final KnowledgeAttachMapper knowledgeAttachMapper;
|
||||
|
||||
public QdrantVectorStoreStrategy(VectorStoreProperties vectorStoreProperties,
|
||||
IChatModelService chatModelService,
|
||||
EmbeddingModelFactory embeddingModelFactory) {
|
||||
EmbeddingModelFactory embeddingModelFactory,
|
||||
KnowledgeAttachMapper knowledgeAttachMapper) {
|
||||
super(vectorStoreProperties, embeddingModelFactory, chatModelService);
|
||||
this.knowledgeAttachMapper = knowledgeAttachMapper;
|
||||
}
|
||||
|
||||
private EmbeddingStore<TextSegment> getQdrantStore(String collectionName) {
|
||||
@@ -129,7 +137,10 @@ public class QdrantVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
||||
metadata.put(METADATA_DOC_ID_KEY, docId);
|
||||
TextSegment textSegment = TextSegment.from(text, metadata);
|
||||
Embedding embedding = embeddingModel.embed(text).content();
|
||||
embeddingStore.add(embedding, textSegment);
|
||||
// 单位化处理
|
||||
float[] vector = embedding.vector();
|
||||
normalize(vector);
|
||||
embeddingStore.add(Embedding.from(vector), textSegment);
|
||||
});
|
||||
|
||||
long endTime = System.currentTimeMillis();
|
||||
@@ -140,18 +151,22 @@ public class QdrantVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
||||
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
|
||||
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName());
|
||||
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
|
||||
// 查询向量单位化处理
|
||||
float[] queryVector = queryEmbedding.vector();
|
||||
normalize(queryVector);
|
||||
|
||||
String collectionName = vectorStoreProperties.getQdrant().getCollectionname() + queryVectorBo.getKid();
|
||||
|
||||
List<Float> vector = new ArrayList<>();
|
||||
for (float f : queryEmbedding.vector()) {
|
||||
vector.add(f);
|
||||
List<Float> vectorList = new ArrayList<>();
|
||||
for (float f : queryVector) {
|
||||
vectorList.add(f);
|
||||
}
|
||||
|
||||
try (QdrantClient client = buildQdrantClient()) {
|
||||
QueryPoints request = QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(Query.newBuilder()
|
||||
.setNearest(vectorInput(vector))
|
||||
.setNearest(vectorInput(vectorList))
|
||||
.build())
|
||||
.setLimit(queryVectorBo.getMaxResults())
|
||||
.setWithPayload(enable(true))
|
||||
@@ -172,6 +187,69 @@ public class QdrantVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KnowledgeRetrievalVo> search(QueryVectorBo queryVectorBo) {
|
||||
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName());
|
||||
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
|
||||
// 查询向量单位化处理
|
||||
float[] queryVector = queryEmbedding.vector();
|
||||
normalize(queryVector);
|
||||
|
||||
String collectionName = vectorStoreProperties.getQdrant().getCollectionname() + queryVectorBo.getKid();
|
||||
|
||||
List<Float> vectorList = new ArrayList<>();
|
||||
for (float f : queryVector) {
|
||||
vectorList.add(f);
|
||||
}
|
||||
|
||||
try (QdrantClient client = buildQdrantClient()) {
|
||||
QueryPoints request = QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(Query.newBuilder()
|
||||
.setNearest(vectorInput(vectorList))
|
||||
.build())
|
||||
.setLimit(queryVectorBo.getMaxResults())
|
||||
.setWithPayload(enable(true))
|
||||
.build();
|
||||
|
||||
List<ScoredPoint> results = client.queryAsync(request).get();
|
||||
List<org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo> resultList = new ArrayList<>();
|
||||
for (ScoredPoint point : results) {
|
||||
String content = "";
|
||||
JsonWithInt.Value textValue = point.getPayloadMap().get(TEXT_SEGMENT_KEY);
|
||||
if (textValue != null && textValue.hasStringValue()) {
|
||||
content = textValue.getStringValue();
|
||||
}
|
||||
|
||||
String docId = null;
|
||||
JsonWithInt.Value docIdValue = point.getPayloadMap().get(METADATA_DOC_ID_KEY);
|
||||
if (docIdValue != null && docIdValue.hasStringValue()) {
|
||||
docId = docIdValue.getStringValue();
|
||||
}
|
||||
|
||||
String sourceName = "未知来源";
|
||||
if (docId != null) {
|
||||
KnowledgeAttach attach = knowledgeAttachMapper.selectOne(new LambdaQueryWrapper<KnowledgeAttach>()
|
||||
.eq(KnowledgeAttach::getDocId, docId)
|
||||
.last("limit 1"));
|
||||
if (attach != null) {
|
||||
sourceName = attach.getName();
|
||||
}
|
||||
}
|
||||
|
||||
resultList.add(org.ruoyi.domain.vo.knowledge.KnowledgeRetrievalVo.builder()
|
||||
.content(content)
|
||||
.score((double) point.getScore())
|
||||
.sourceName(sourceName)
|
||||
.build());
|
||||
}
|
||||
return resultList;
|
||||
} catch (Exception e) {
|
||||
log.error("Qdrant检索失败: {}", collectionName, e);
|
||||
throw new ServiceException("Qdrant向量检索失败");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeById(String id, String modelName) {
|
||||
String collectionName = vectorStoreProperties.getQdrant().getCollectionname() + id;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user