mirror of
https://gitcode.com/ageerle/ruoyi-ai.git
synced 2026-04-16 21:33:40 +00:00
@@ -17,7 +17,7 @@ public class VectorStoreProperties {
|
|||||||
/**
|
/**
|
||||||
* 向量库类型
|
* 向量库类型
|
||||||
*/
|
*/
|
||||||
private String type = "weaviate";
|
private String type;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Weaviate配置
|
* Weaviate配置
|
||||||
@@ -34,17 +34,17 @@ public class VectorStoreProperties {
|
|||||||
/**
|
/**
|
||||||
* 协议
|
* 协议
|
||||||
*/
|
*/
|
||||||
private String protocol = "http";
|
private String protocol;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 主机地址
|
* 主机地址
|
||||||
*/
|
*/
|
||||||
private String host = "localhost:8080";
|
private String host;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 类名
|
* 类名
|
||||||
*/
|
*/
|
||||||
private String classname = "Document";
|
private String classname;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
@@ -52,11 +52,11 @@ public class VectorStoreProperties {
|
|||||||
/**
|
/**
|
||||||
* 连接URL
|
* 连接URL
|
||||||
*/
|
*/
|
||||||
private String url = "http://localhost:19530";
|
private String url;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 集合名称
|
* 集合名称
|
||||||
*/
|
*/
|
||||||
private String collectionname = "knowledge_base";
|
private String collectionname;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -70,6 +70,11 @@ public class ChatModelVo implements Serializable {
|
|||||||
@ExcelProperty(value = "是否显示")
|
@ExcelProperty(value = "是否显示")
|
||||||
private String modelShow;
|
private String modelShow;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 模型维度
|
||||||
|
*/
|
||||||
|
private Integer dimension;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 系统提示词
|
* 系统提示词
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -80,6 +80,12 @@
|
|||||||
<version>2.6.4</version>
|
<version>2.6.4</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<!-- LangChain4j Milvus Embedding Store -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>dev.langchain4j</groupId>
|
||||||
|
<artifactId>langchain4j-milvus</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>dev.langchain4j</groupId>
|
<groupId>dev.langchain4j</groupId>
|
||||||
<artifactId>langchain4j-open-ai</artifactId>
|
<artifactId>langchain4j-open-ai</artifactId>
|
||||||
|
|||||||
@@ -32,9 +32,9 @@ public class StoreEmbeddingBo {
|
|||||||
private List<String> fids;
|
private List<String> fids;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 向量库模型名称
|
* 向量库名称
|
||||||
*/
|
*/
|
||||||
private String vectorModelName;
|
private String vectorStoreName;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 向量化模型id
|
* 向量化模型id
|
||||||
|
|||||||
@@ -27,20 +27,23 @@ public class EmbeddingModelFactory {
|
|||||||
private final IChatModelService chatModelService;
|
private final IChatModelService chatModelService;
|
||||||
|
|
||||||
// 模型缓存,使用ConcurrentHashMap保证线程安全
|
// 模型缓存,使用ConcurrentHashMap保证线程安全
|
||||||
private final Map<Long, BaseEmbedModelService> modelCache = new ConcurrentHashMap<>();
|
private final Map<String, BaseEmbedModelService> modelCache = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建嵌入模型实例
|
* 创建嵌入模型实例
|
||||||
* 如果模型已存在于缓存中,则直接返回;否则创建新的实例
|
* 如果模型已存在于缓存中,则直接返回;否则创建新的实例
|
||||||
*
|
*
|
||||||
* @param embeddingModelId 嵌入模型的唯一标识ID
|
* @param embeddingModelName 嵌入模型名称
|
||||||
* @return BaseEmbedModelService 嵌入模型服务实例
|
* @param dimension 模型维度大小
|
||||||
*/
|
*/
|
||||||
public BaseEmbedModelService createModel(Long embeddingModelId) {
|
public BaseEmbedModelService createModel(String embeddingModelName, Integer dimension) {
|
||||||
return modelCache.computeIfAbsent(embeddingModelId, id -> {
|
return modelCache.computeIfAbsent(embeddingModelName, name -> {
|
||||||
ChatModelVo modelConfig = chatModelService.queryById(id);
|
ChatModelVo modelConfig = chatModelService.selectModelByName(embeddingModelName);
|
||||||
if (modelConfig == null) {
|
if (modelConfig == null) {
|
||||||
throw new IllegalArgumentException("未找到模型配置,ID=" + id);
|
throw new IllegalArgumentException("未找到模型配置,name=" + name);
|
||||||
|
}
|
||||||
|
if (modelConfig.getDimension() != null) {
|
||||||
|
modelConfig.setDimension(dimension);
|
||||||
}
|
}
|
||||||
return createModelInstance(modelConfig.getProviderName(), modelConfig);
|
return createModelInstance(modelConfig.getProviderName(), modelConfig);
|
||||||
});
|
});
|
||||||
@@ -49,22 +52,22 @@ public class EmbeddingModelFactory {
|
|||||||
/**
|
/**
|
||||||
* 检查模型是否支持多模态
|
* 检查模型是否支持多模态
|
||||||
*
|
*
|
||||||
* @param embeddingModelId 嵌入模型的唯一标识ID
|
* @param embeddingModelName 嵌入模型名称
|
||||||
* @return boolean 如果模型支持多模态则返回true,否则返回false
|
* @return boolean 如果模型支持多模态则返回true,否则返回false
|
||||||
*/
|
*/
|
||||||
public boolean isMultimodalModel(Long embeddingModelId) {
|
public boolean isMultimodalModel(String embeddingModelName) {
|
||||||
return createModel(embeddingModelId) instanceof MultiModalEmbedModelService;
|
return createModel(embeddingModelName, null) instanceof MultiModalEmbedModelService;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建多模态嵌入模型实例
|
* 创建多模态嵌入模型实例
|
||||||
*
|
*
|
||||||
* @param tenantId 租户ID
|
* @param embeddingModelName 嵌入模型名称
|
||||||
* @return MultiModalEmbedModelService 多模态嵌入模型服务实例
|
* @return MultiModalEmbedModelService 多模态嵌入模型服务实例
|
||||||
* @throws IllegalArgumentException 当模型不支持多模态时抛出
|
* @throws IllegalArgumentException 当模型不支持多模态时抛出
|
||||||
*/
|
*/
|
||||||
public MultiModalEmbedModelService createMultimodalModel(Long tenantId) {
|
public MultiModalEmbedModelService createMultimodalModel(String embeddingModelName) {
|
||||||
BaseEmbedModelService model = createModel(tenantId);
|
BaseEmbedModelService model = createModel(embeddingModelName, null);
|
||||||
if (model instanceof MultiModalEmbedModelService) {
|
if (model instanceof MultiModalEmbedModelService) {
|
||||||
return (MultiModalEmbedModelService) model;
|
return (MultiModalEmbedModelService) model;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ public class OllamaEmbeddingProvider implements BaseEmbedModelService {
|
|||||||
return Set.of(ModalityType.TEXT);
|
return Set.of(ModalityType.TEXT);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ollama不能设置embedding维度,使用milvus时请注意!!创建向量表时需要先设定维度大小
|
||||||
@Override
|
@Override
|
||||||
public Response<List<Embedding>> embedAll(List<TextSegment> textSegments) {
|
public Response<List<Embedding>> embedAll(List<TextSegment> textSegments) {
|
||||||
return OllamaEmbeddingModel.builder()
|
return OllamaEmbeddingModel.builder()
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ public class OpenAiEmbeddingProvider implements BaseEmbedModelService {
|
|||||||
.baseUrl(chatModelVo.getApiHost())
|
.baseUrl(chatModelVo.getApiHost())
|
||||||
.apiKey(chatModelVo.getApiKey())
|
.apiKey(chatModelVo.getApiKey())
|
||||||
.modelName(chatModelVo.getModelName())
|
.modelName(chatModelVo.getModelName())
|
||||||
|
.dimensions(chatModelVo.getDimension())
|
||||||
.build()
|
.build()
|
||||||
.embedAll(textSegments);
|
.embedAll(textSegments);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ public class ZhiPuAiEmbeddingProvider implements BaseEmbedModelService {
|
|||||||
.baseUrl(chatModelVo.getApiHost())
|
.baseUrl(chatModelVo.getApiHost())
|
||||||
.apiKey(chatModelVo.getApiKey())
|
.apiKey(chatModelVo.getApiKey())
|
||||||
.model(chatModelVo.getModelName())
|
.model(chatModelVo.getModelName())
|
||||||
|
.dimensions(chatModelVo.getDimension())
|
||||||
.build()
|
.build()
|
||||||
.embedAll(textSegments);
|
.embedAll(textSegments);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ public interface VectorStoreService {
|
|||||||
|
|
||||||
List<String> getQueryVector(QueryVectorBo queryVectorBo);
|
List<String> getQueryVector(QueryVectorBo queryVectorBo);
|
||||||
|
|
||||||
void createSchema(String vectorModelName, String kid,String modelName);
|
void createSchema(String kid, String embeddingModelName);
|
||||||
|
|
||||||
void removeById(String id,String modelName) throws ServiceException;
|
void removeById(String id,String modelName) throws ServiceException;
|
||||||
|
|
||||||
|
|||||||
@@ -2,16 +2,13 @@ package org.ruoyi.service.impl;
|
|||||||
|
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.ruoyi.common.core.service.ConfigService;
|
|
||||||
import org.ruoyi.domain.bo.QueryVectorBo;
|
import org.ruoyi.domain.bo.QueryVectorBo;
|
||||||
import org.ruoyi.domain.bo.StoreEmbeddingBo;
|
import org.ruoyi.domain.bo.StoreEmbeddingBo;
|
||||||
import org.ruoyi.service.VectorStoreService;
|
import org.ruoyi.service.VectorStoreService;
|
||||||
import org.ruoyi.service.strategy.VectorStoreStrategy;
|
|
||||||
import org.ruoyi.service.strategy.VectorStoreStrategyFactory;
|
import org.ruoyi.service.strategy.VectorStoreStrategyFactory;
|
||||||
import org.springframework.context.annotation.Primary;
|
import org.springframework.context.annotation.Primary;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 向量库管理
|
* 向量库管理
|
||||||
@@ -30,22 +27,21 @@ public class VectorStoreServiceImpl implements VectorStoreService {
|
|||||||
/**
|
/**
|
||||||
* 获取当前配置的向量库策略
|
* 获取当前配置的向量库策略
|
||||||
*/
|
*/
|
||||||
private VectorStoreStrategy getCurrentStrategy() {
|
private VectorStoreService getCurrentStrategy() {
|
||||||
return strategyFactory.getStrategy();
|
return strategyFactory.getStrategy();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void createSchema(String vectorModelName, String kid, String modelName) {
|
public void createSchema(String kid, String modelName) {
|
||||||
log.info("创建向量库schema: vectorModelName={}, kid={}, modelName={}", vectorModelName, kid, modelName);
|
VectorStoreService strategy = getCurrentStrategy();
|
||||||
VectorStoreStrategy strategy = getCurrentStrategy();
|
strategy.createSchema(kid, modelName);
|
||||||
strategy.createSchema(vectorModelName, kid, modelName);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
|
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
|
||||||
log.info("存储向量数据: kid={}, docId={}, 数据条数={}",
|
log.info("存储向量数据: kid={}, docId={}, 数据条数={}",
|
||||||
storeEmbeddingBo.getKid(), storeEmbeddingBo.getDocId(), storeEmbeddingBo.getChunkList().size());
|
storeEmbeddingBo.getKid(), storeEmbeddingBo.getDocId(), storeEmbeddingBo.getChunkList().size());
|
||||||
VectorStoreStrategy strategy = getCurrentStrategy();
|
VectorStoreService strategy = getCurrentStrategy();
|
||||||
strategy.storeEmbeddings(storeEmbeddingBo);
|
strategy.storeEmbeddings(storeEmbeddingBo);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,28 +49,28 @@ public class VectorStoreServiceImpl implements VectorStoreService {
|
|||||||
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
|
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
|
||||||
log.info("查询向量数据: kid={}, query={}, maxResults={}",
|
log.info("查询向量数据: kid={}, query={}, maxResults={}",
|
||||||
queryVectorBo.getKid(), queryVectorBo.getQuery(), queryVectorBo.getMaxResults());
|
queryVectorBo.getKid(), queryVectorBo.getQuery(), queryVectorBo.getMaxResults());
|
||||||
VectorStoreStrategy strategy = getCurrentStrategy();
|
VectorStoreService strategy = getCurrentStrategy();
|
||||||
return strategy.getQueryVector(queryVectorBo);
|
return strategy.getQueryVector(queryVectorBo);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void removeById(String id, String modelName) {
|
public void removeById(String id, String modelName) {
|
||||||
log.info("根据ID删除向量数据: id={}, modelName={}", id, modelName);
|
log.info("根据ID删除向量数据: id={}, modelName={}", id, modelName);
|
||||||
VectorStoreStrategy strategy = getCurrentStrategy();
|
VectorStoreService strategy = getCurrentStrategy();
|
||||||
strategy.removeById(id, modelName);
|
strategy.removeById(id, modelName);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void removeByDocId(String docId, String kid) {
|
public void removeByDocId(String docId, String kid) {
|
||||||
log.info("根据docId删除向量数据: docId={}, kid={}", docId, kid);
|
log.info("根据docId删除向量数据: docId={}, kid={}", docId, kid);
|
||||||
VectorStoreStrategy strategy = getCurrentStrategy();
|
VectorStoreService strategy = getCurrentStrategy();
|
||||||
strategy.removeByDocId(docId, kid);
|
strategy.removeByDocId(docId, kid);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void removeByFid(String fid, String kid) {
|
public void removeByFid(String fid, String kid) {
|
||||||
log.info("根据fid删除向量数据: fid={}, kid={}", fid, kid);
|
log.info("根据fid删除向量数据: fid={}, kid={}", fid, kid);
|
||||||
VectorStoreStrategy strategy = getCurrentStrategy();
|
VectorStoreService strategy = getCurrentStrategy();
|
||||||
strategy.removeByFid(fid, kid);
|
strategy.removeByFid(fid, kid);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,40 +8,30 @@ import lombok.RequiredArgsConstructor;
|
|||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.ruoyi.common.core.config.VectorStoreProperties;
|
import org.ruoyi.common.core.config.VectorStoreProperties;
|
||||||
|
import org.ruoyi.common.core.utils.StringUtils;
|
||||||
|
import org.ruoyi.service.VectorStoreService;
|
||||||
|
import org.ruoyi.embedding.EmbeddingModelFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 向量库策略抽象基类
|
* 向量库策略抽象基类
|
||||||
* 提供公共的方法实现,如embedding模型获取等
|
* 提供公共的方法实现,如embedding模型获取等
|
||||||
*
|
*
|
||||||
* @author ageer
|
* @author Yzm
|
||||||
*/
|
*/
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public abstract class AbstractVectorStoreStrategy implements VectorStoreStrategy {
|
public abstract class AbstractVectorStoreStrategy implements VectorStoreService {
|
||||||
|
|
||||||
protected final VectorStoreProperties vectorStoreProperties;
|
protected final VectorStoreProperties vectorStoreProperties;
|
||||||
|
|
||||||
|
private final EmbeddingModelFactory embeddingModelFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取向量模型
|
* 获取向量模型
|
||||||
*/
|
*/
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
protected EmbeddingModel getEmbeddingModel(String modelName, String apiKey, String baseUrl) {
|
protected EmbeddingModel getEmbeddingModel(String modelName, Integer dimension) {
|
||||||
EmbeddingModel embeddingModel;
|
return embeddingModelFactory.createModel(modelName, dimension);
|
||||||
if ("quentinz/bge-large-zh-v1.5".equals(modelName)) {
|
|
||||||
embeddingModel = OllamaEmbeddingModel.builder()
|
|
||||||
.baseUrl(baseUrl)
|
|
||||||
.modelName(modelName)
|
|
||||||
.build();
|
|
||||||
} else if ("baai/bge-m3".equals(modelName)) {
|
|
||||||
embeddingModel = OpenAiEmbeddingModel.builder()
|
|
||||||
.apiKey(apiKey)
|
|
||||||
.baseUrl(baseUrl)
|
|
||||||
.modelName(modelName)
|
|
||||||
.build();
|
|
||||||
} else {
|
|
||||||
throw new ServiceException("未找到对应向量化模型!");
|
|
||||||
}
|
|
||||||
return embeddingModel;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
package org.ruoyi.service.strategy;
|
|
||||||
|
|
||||||
import org.ruoyi.service.VectorStoreService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 向量库策略接口
|
|
||||||
* 继承VectorStoreService以避免重复定义相同的方法
|
|
||||||
*
|
|
||||||
* @author ageer
|
|
||||||
*/
|
|
||||||
public interface VectorStoreStrategy extends VectorStoreService {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取向量库类型标识
|
|
||||||
* @return 向量库类型(如:weaviate, milvus)
|
|
||||||
*/
|
|
||||||
String getVectorStoreType();
|
|
||||||
}
|
|
||||||
@@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.ruoyi.common.core.config.VectorStoreProperties;
|
import org.ruoyi.common.core.config.VectorStoreProperties;
|
||||||
import org.ruoyi.service.strategy.impl.MilvusVectorStoreStrategy;
|
import org.ruoyi.service.strategy.impl.MilvusVectorStoreStrategy;
|
||||||
import org.ruoyi.service.strategy.impl.WeaviateVectorStoreStrategy;
|
import org.ruoyi.service.strategy.impl.WeaviateVectorStoreStrategy;
|
||||||
|
import org.ruoyi.service.VectorStoreService;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@@ -15,7 +16,7 @@ import java.util.Map;
|
|||||||
* 向量库策略工厂
|
* 向量库策略工厂
|
||||||
* 根据配置动态选择向量库实现
|
* 根据配置动态选择向量库实现
|
||||||
*
|
*
|
||||||
* @author ageer
|
* @author Yzm
|
||||||
*/
|
*/
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
@@ -26,7 +27,7 @@ public class VectorStoreStrategyFactory {
|
|||||||
private final WeaviateVectorStoreStrategy weaviateStrategy;
|
private final WeaviateVectorStoreStrategy weaviateStrategy;
|
||||||
private final MilvusVectorStoreStrategy milvusStrategy;
|
private final MilvusVectorStoreStrategy milvusStrategy;
|
||||||
|
|
||||||
private Map<String, VectorStoreStrategy> strategies;
|
private Map<String, VectorStoreService> strategies;
|
||||||
|
|
||||||
@PostConstruct
|
@PostConstruct
|
||||||
public void init() {
|
public void init() {
|
||||||
@@ -39,36 +40,18 @@ public class VectorStoreStrategyFactory {
|
|||||||
/**
|
/**
|
||||||
* 获取当前配置的向量库策略
|
* 获取当前配置的向量库策略
|
||||||
*/
|
*/
|
||||||
public VectorStoreStrategy getStrategy() {
|
public VectorStoreService getStrategy() {
|
||||||
String vectorStoreType = vectorStoreProperties.getType();
|
String vectorStoreType = vectorStoreProperties.getType();
|
||||||
if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) {
|
if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) {
|
||||||
vectorStoreType = "weaviate"; // 默认使用weaviate
|
vectorStoreType = "weaviate"; // 默认使用weaviate
|
||||||
}
|
}
|
||||||
|
VectorStoreService strategy = strategies.get(vectorStoreType.toLowerCase());
|
||||||
VectorStoreStrategy strategy = strategies.get(vectorStoreType.toLowerCase());
|
|
||||||
if (strategy == null) {
|
if (strategy == null) {
|
||||||
log.warn("未找到向量库策略: {}, 使用默认策略: weaviate", vectorStoreType);
|
log.warn("未找到向量库策略: {}, 使用默认策略: weaviate", vectorStoreType);
|
||||||
strategy = strategies.get("weaviate");
|
strategy = strategies.get("weaviate");
|
||||||
}
|
}
|
||||||
|
|
||||||
log.debug("使用向量库策略: {}", vectorStoreType);
|
log.debug("使用向量库策略: {}", vectorStoreType);
|
||||||
return strategy;
|
return strategy;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 根据类型获取向量库策略
|
|
||||||
*/
|
|
||||||
public VectorStoreStrategy getStrategy(String type) {
|
|
||||||
if (type == null || type.trim().isEmpty()) {
|
|
||||||
return getStrategy();
|
|
||||||
}
|
|
||||||
|
|
||||||
VectorStoreStrategy strategy = strategies.get(type.toLowerCase());
|
|
||||||
if (strategy == null) {
|
|
||||||
log.warn("未找到向量库策略: {}, 使用默认策略", type);
|
|
||||||
return getStrategy();
|
|
||||||
}
|
|
||||||
|
|
||||||
return strategy;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -1,159 +1,75 @@
|
|||||||
package org.ruoyi.service.strategy.impl;
|
package org.ruoyi.service.strategy.impl;
|
||||||
|
|
||||||
import org.ruoyi.common.core.exception.ServiceException;
|
import dev.langchain4j.data.document.Metadata;
|
||||||
import dev.langchain4j.data.embedding.Embedding;
|
import dev.langchain4j.data.embedding.Embedding;
|
||||||
|
import dev.langchain4j.data.segment.TextSegment;
|
||||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||||
import io.milvus.client.MilvusServiceClient;
|
import dev.langchain4j.store.embedding.EmbeddingMatch;
|
||||||
import io.milvus.common.clientenum.ConsistencyLevelEnum;
|
import dev.langchain4j.store.embedding.EmbeddingSearchRequest;
|
||||||
import io.milvus.grpc.*;
|
import dev.langchain4j.store.embedding.EmbeddingStore;
|
||||||
import io.milvus.param.*;
|
import dev.langchain4j.store.embedding.filter.Filter;
|
||||||
import io.milvus.param.collection.*;
|
import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder;
|
||||||
import io.milvus.param.dml.DeleteParam;
|
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
|
||||||
import io.milvus.param.dml.InsertParam;
|
import io.milvus.param.IndexType;
|
||||||
import io.milvus.param.dml.SearchParam;
|
import io.milvus.param.MetricType;
|
||||||
import io.milvus.param.index.CreateIndexParam;
|
|
||||||
import io.milvus.param.index.DescribeIndexParam;
|
|
||||||
import io.milvus.response.DescCollResponseWrapper;
|
|
||||||
import io.milvus.response.SearchResultsWrapper;
|
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.ruoyi.common.core.config.VectorStoreProperties;
|
import org.ruoyi.common.core.config.VectorStoreProperties;
|
||||||
import org.ruoyi.domain.bo.QueryVectorBo;
|
import org.ruoyi.domain.bo.QueryVectorBo;
|
||||||
import org.ruoyi.domain.bo.StoreEmbeddingBo;
|
import org.ruoyi.domain.bo.StoreEmbeddingBo;
|
||||||
|
import org.ruoyi.embedding.EmbeddingModelFactory;
|
||||||
import org.ruoyi.service.strategy.AbstractVectorStoreStrategy;
|
import org.ruoyi.service.strategy.AbstractVectorStoreStrategy;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.stream.IntStream;
|
||||||
|
|
||||||
/**
|
|
||||||
* Milvus向量库策略实现
|
|
||||||
*
|
|
||||||
* @author ageer
|
|
||||||
*/
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
||||||
|
|
||||||
private MilvusServiceClient milvusClient;
|
|
||||||
|
|
||||||
public MilvusVectorStoreStrategy(VectorStoreProperties vectorStoreProperties) {
|
private final Integer DIMENSION = 2048;
|
||||||
super(vectorStoreProperties);
|
|
||||||
|
public MilvusVectorStoreStrategy(VectorStoreProperties vectorStoreProperties, EmbeddingModelFactory embeddingModelFactory) {
|
||||||
|
super(vectorStoreProperties, embeddingModelFactory);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 缓存不同集合与 autoFlush 配置的 Milvus 连接
|
||||||
|
private final Map<String, EmbeddingStore<TextSegment>> storeCache = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
|
private EmbeddingStore<TextSegment> getMilvusStore(String collectionName, boolean autoFlushOnInsert) {
|
||||||
|
String key = collectionName + "|" + autoFlushOnInsert;
|
||||||
|
return storeCache.computeIfAbsent(key, k ->
|
||||||
|
MilvusEmbeddingStore.builder()
|
||||||
|
.uri(vectorStoreProperties.getMilvus().getUrl())
|
||||||
|
.collectionName(collectionName)
|
||||||
|
.dimension(DIMENSION)
|
||||||
|
.indexType(IndexType.IVF_FLAT)
|
||||||
|
.metricType(MetricType.L2)
|
||||||
|
.autoFlushOnInsert(autoFlushOnInsert)
|
||||||
|
.idFieldName("id")
|
||||||
|
.textFieldName("text")
|
||||||
|
.metadataFieldName("metadata")
|
||||||
|
.vectorFieldName("vector")
|
||||||
|
.build()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getVectorStoreType() {
|
public void createSchema(String kid, String modelName) {
|
||||||
return "milvus";
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void createSchema(String vectorModelName, String kid, String modelName) {
|
|
||||||
String url = vectorStoreProperties.getMilvus().getUrl();
|
|
||||||
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
|
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
|
||||||
|
// 使用缓存获取连接以确保只初始化一次
|
||||||
// 创建Milvus客户端连接
|
EmbeddingStore<TextSegment> store = getMilvusStore(collectionName, true);
|
||||||
ConnectParam connectParam = ConnectParam.newBuilder()
|
log.info("Milvus集合初始化完成: {}", collectionName);
|
||||||
.withUri(url)
|
|
||||||
.build();
|
|
||||||
milvusClient = new MilvusServiceClient(connectParam);
|
|
||||||
|
|
||||||
// 检查集合是否存在
|
|
||||||
HasCollectionParam hasCollectionParam = HasCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<Boolean> hasCollectionResponse = milvusClient.hasCollection(hasCollectionParam);
|
|
||||||
if (hasCollectionResponse.getStatus() != R.Status.Success.getCode()) {
|
|
||||||
log.error("检查集合是否存在失败: {}", hasCollectionResponse.getMessage());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!hasCollectionResponse.getData()) {
|
|
||||||
// 创建字段
|
|
||||||
List<FieldType> fields = new ArrayList<>();
|
|
||||||
|
|
||||||
// ID字段 (主键)
|
|
||||||
fields.add(FieldType.newBuilder()
|
|
||||||
.withName("id")
|
|
||||||
.withDataType(DataType.Int64)
|
|
||||||
.withPrimaryKey(true)
|
|
||||||
.withAutoID(true)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
// 文本字段
|
|
||||||
fields.add(FieldType.newBuilder()
|
|
||||||
.withName("text")
|
|
||||||
.withDataType(DataType.VarChar)
|
|
||||||
.withMaxLength(65535)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
// fid字段
|
|
||||||
fields.add(FieldType.newBuilder()
|
|
||||||
.withName("fid")
|
|
||||||
.withDataType(DataType.VarChar)
|
|
||||||
.withMaxLength(255)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
// kid字段
|
|
||||||
fields.add(FieldType.newBuilder()
|
|
||||||
.withName("kid")
|
|
||||||
.withDataType(DataType.VarChar)
|
|
||||||
.withMaxLength(255)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
// docId字段
|
|
||||||
fields.add(FieldType.newBuilder()
|
|
||||||
.withName("docId")
|
|
||||||
.withDataType(DataType.VarChar)
|
|
||||||
.withMaxLength(255)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
// 向量字段
|
|
||||||
fields.add(FieldType.newBuilder()
|
|
||||||
.withName("vector")
|
|
||||||
.withDataType(DataType.FloatVector)
|
|
||||||
.withDimension(1024) // 根据实际embedding维度调整
|
|
||||||
.build());
|
|
||||||
|
|
||||||
// 创建集合
|
|
||||||
CreateCollectionParam createCollectionParam = CreateCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.withDescription("Knowledge base collection for " + kid)
|
|
||||||
.withShardsNum(2)
|
|
||||||
.withFieldTypes(fields)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<RpcStatus> createCollectionResponse = milvusClient.createCollection(createCollectionParam);
|
|
||||||
if (createCollectionResponse.getStatus() != R.Status.Success.getCode()) {
|
|
||||||
log.error("创建集合失败: {}", createCollectionResponse.getMessage());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 创建索引
|
|
||||||
CreateIndexParam createIndexParam = CreateIndexParam.newBuilder()
|
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.withFieldName("vector")
|
|
||||||
.withIndexType(IndexType.IVF_FLAT)
|
|
||||||
.withMetricType(MetricType.L2)
|
|
||||||
.withExtraParam("{\"nlist\":1024}")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<RpcStatus> createIndexResponse = milvusClient.createIndex(createIndexParam);
|
|
||||||
if (createIndexResponse.getStatus() != R.Status.Success.getCode()) {
|
|
||||||
log.error("创建索引失败: {}", createIndexResponse.getMessage());
|
|
||||||
} else {
|
|
||||||
log.info("Milvus集合和索引创建成功: {}", collectionName);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
log.info("Milvus集合已存在: {}", collectionName);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
|
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
|
||||||
createSchema(storeEmbeddingBo.getVectorModelName(), storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName());
|
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), DIMENSION);
|
||||||
|
|
||||||
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(),
|
|
||||||
storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl());
|
|
||||||
|
|
||||||
List<String> chunkList = storeEmbeddingBo.getChunkList();
|
List<String> chunkList = storeEmbeddingBo.getChunkList();
|
||||||
List<String> fidList = storeEmbeddingBo.getFids();
|
List<String> fidList = storeEmbeddingBo.getFids();
|
||||||
@@ -161,177 +77,81 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
|||||||
String docId = storeEmbeddingBo.getDocId();
|
String docId = storeEmbeddingBo.getDocId();
|
||||||
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
|
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
|
||||||
|
|
||||||
log.info("Milvus向量存储条数记录: " + chunkList.size());
|
log.info("Milvus向量存储条数记录: {}", chunkList.size());
|
||||||
long startTime = System.currentTimeMillis();
|
long startTime = System.currentTimeMillis();
|
||||||
|
|
||||||
// 准备批量插入数据
|
// 复用连接,写入场景使用 autoFlush=false 以提升批量插入性能
|
||||||
List<InsertParam.Field> fields = new ArrayList<>();
|
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, false);
|
||||||
List<String> textList = new ArrayList<>();
|
|
||||||
List<String> fidListData = new ArrayList<>();
|
|
||||||
List<String> kidList = new ArrayList<>();
|
|
||||||
List<String> docIdList = new ArrayList<>();
|
|
||||||
List<List<Float>> vectorList = new ArrayList<>();
|
|
||||||
|
|
||||||
for (int i = 0; i < chunkList.size(); i++) {
|
IntStream.range(0, chunkList.size()).forEach(i -> {
|
||||||
String text = chunkList.get(i);
|
String text = chunkList.get(i);
|
||||||
String fid = fidList.get(i);
|
String fid = fidList.get(i);
|
||||||
|
Metadata metadata = new Metadata();
|
||||||
|
metadata.put("fid", fid);
|
||||||
|
metadata.put("kid", kid);
|
||||||
|
metadata.put("docId", docId);
|
||||||
|
|
||||||
|
TextSegment textSegment = TextSegment.from(text, metadata);
|
||||||
Embedding embedding = embeddingModel.embed(text).content();
|
Embedding embedding = embeddingModel.embed(text).content();
|
||||||
|
embeddingStore.add(embedding, textSegment);
|
||||||
textList.add(text);
|
});
|
||||||
fidListData.add(fid);
|
|
||||||
kidList.add(kid);
|
|
||||||
docIdList.add(docId);
|
|
||||||
|
|
||||||
List<Float> vector = new ArrayList<>();
|
|
||||||
for (float f : embedding.vector()) {
|
|
||||||
vector.add(f);
|
|
||||||
}
|
|
||||||
vectorList.add(vector);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 构建字段数据
|
|
||||||
fields.add(new InsertParam.Field("text", textList));
|
|
||||||
fields.add(new InsertParam.Field("fid", fidListData));
|
|
||||||
fields.add(new InsertParam.Field("kid", kidList));
|
|
||||||
fields.add(new InsertParam.Field("docId", docIdList));
|
|
||||||
fields.add(new InsertParam.Field("vector", vectorList));
|
|
||||||
|
|
||||||
// 执行插入
|
|
||||||
InsertParam insertParam = InsertParam.newBuilder()
|
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.withFields(fields)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<MutationResult> insertResponse = milvusClient.insert(insertParam);
|
|
||||||
if (insertResponse.getStatus() != R.Status.Success.getCode()) {
|
|
||||||
log.error("Milvus向量存储失败: {}", insertResponse.getMessage());
|
|
||||||
throw new ServiceException("Milvus向量存储失败");
|
|
||||||
} else {
|
|
||||||
log.info("Milvus向量存储成功,插入条数: {}", insertResponse.getData().getInsertCnt());
|
|
||||||
}
|
|
||||||
|
|
||||||
long endTime = System.currentTimeMillis();
|
long endTime = System.currentTimeMillis();
|
||||||
log.info("Milvus向量存储完成消耗时间:" + (endTime - startTime) / 1000 + "秒");
|
log.info("Milvus向量存储完成消耗时间:{}秒", (endTime - startTime) / 1000);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
|
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
|
||||||
createSchema(queryVectorBo.getVectorModelName(), queryVectorBo.getKid(), queryVectorBo.getVectorModelName());
|
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(), DIMENSION);
|
||||||
|
|
||||||
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(),
|
|
||||||
queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl());
|
|
||||||
|
|
||||||
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
|
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
|
||||||
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + queryVectorBo.getKid();
|
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + queryVectorBo.getKid();
|
||||||
|
|
||||||
|
// 查询复用连接,autoFlush 对查询无影响,此处保持 true
|
||||||
|
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, true);
|
||||||
|
|
||||||
List<String> resultList = new ArrayList<>();
|
List<String> resultList = new ArrayList<>();
|
||||||
|
EmbeddingSearchRequest request = EmbeddingSearchRequest.builder()
|
||||||
// 加载集合到内存
|
.queryEmbedding(queryEmbedding)
|
||||||
LoadCollectionParam loadCollectionParam = LoadCollectionParam.newBuilder()
|
.maxResults(queryVectorBo.getMaxResults())
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.build();
|
.build();
|
||||||
milvusClient.loadCollection(loadCollectionParam);
|
List<EmbeddingMatch<TextSegment>> matches = embeddingStore.search(request).matches();
|
||||||
|
for (EmbeddingMatch<TextSegment> match : matches) {
|
||||||
// 准备查询向量
|
TextSegment segment = match.embedded();
|
||||||
List<List<Float>> searchVectors = new ArrayList<>();
|
if (segment != null) {
|
||||||
List<Float> queryVector = new ArrayList<>();
|
resultList.add(segment.text());
|
||||||
for (float f : queryEmbedding.vector()) {
|
|
||||||
queryVector.add(f);
|
|
||||||
}
|
|
||||||
searchVectors.add(queryVector);
|
|
||||||
|
|
||||||
// 构建搜索参数
|
|
||||||
SearchParam searchParam = SearchParam.newBuilder()
|
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.withMetricType(MetricType.L2)
|
|
||||||
.withOutFields(Arrays.asList("text", "fid", "kid", "docId"))
|
|
||||||
.withTopK(queryVectorBo.getMaxResults())
|
|
||||||
.withVectors(searchVectors)
|
|
||||||
.withVectorFieldName("vector")
|
|
||||||
.withParams("{\"nprobe\":10}")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<SearchResults> searchResponse = milvusClient.search(searchParam);
|
|
||||||
if (searchResponse.getStatus() != R.Status.Success.getCode()) {
|
|
||||||
log.error("Milvus查询失败: {}", searchResponse.getMessage());
|
|
||||||
return resultList;
|
|
||||||
}
|
|
||||||
|
|
||||||
SearchResultsWrapper wrapper = new SearchResultsWrapper(searchResponse.getData().getResults());
|
|
||||||
|
|
||||||
// 遍历搜索结果
|
|
||||||
for (int i = 0; i < wrapper.getIDScore(0).size(); i++) {
|
|
||||||
SearchResultsWrapper.IDScore idScore = wrapper.getIDScore(0).get(i);
|
|
||||||
|
|
||||||
// 获取text字段数据
|
|
||||||
List<?> textFieldData = wrapper.getFieldData("text", 0);
|
|
||||||
if (textFieldData != null && i < textFieldData.size()) {
|
|
||||||
Object textObj = textFieldData.get(i);
|
|
||||||
if (textObj != null) {
|
|
||||||
resultList.add(textObj.toString());
|
|
||||||
log.debug("找到相似文本,ID: {}, 距离: {}, 内容: {}",
|
|
||||||
idScore.getLongID(), idScore.getScore(), textObj.toString());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return resultList;
|
return resultList;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void removeById(String id, String modelName) {
|
public void removeById(String id, String modelName) {
|
||||||
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + id;
|
// 注意:此处原逻辑使用 collectionname + id,保持现状
|
||||||
|
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(vectorStoreProperties.getMilvus().getCollectionname() + id, false);
|
||||||
// 删除整个集合
|
embeddingStore.remove(id);
|
||||||
DropCollectionParam dropCollectionParam = DropCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<RpcStatus> dropResponse = milvusClient.dropCollection(dropCollectionParam);
|
|
||||||
if (dropResponse.getStatus() != R.Status.Success.getCode()) {
|
|
||||||
log.error("Milvus集合删除失败: {}", dropResponse.getMessage());
|
|
||||||
throw new ServiceException("Milvus集合删除失败");
|
|
||||||
} else {
|
|
||||||
log.info("Milvus集合删除成功: {}", collectionName);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void removeByDocId(String docId, String kid) {
|
public void removeByDocId(String docId, String kid) {
|
||||||
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
|
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
|
||||||
|
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, false);
|
||||||
String expr = "docId == \"" + docId + "\"";
|
Filter filter = MetadataFilterBuilder.metadataKey("docId").isEqualTo(docId);
|
||||||
DeleteParam deleteParam = DeleteParam.newBuilder()
|
embeddingStore.removeAll(filter);
|
||||||
.withCollectionName(collectionName)
|
log.info("Milvus成功删除 docId={} 的所有向量数据", docId);
|
||||||
.withExpr(expr)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<MutationResult> deleteResponse = milvusClient.delete(deleteParam);
|
|
||||||
if (deleteResponse.getStatus() != R.Status.Success.getCode()) {
|
|
||||||
log.error("Milvus删除失败: {}", deleteResponse.getMessage());
|
|
||||||
throw new ServiceException("Milvus删除失败");
|
|
||||||
} else {
|
|
||||||
log.info("Milvus成功删除 docId={} 的所有向量数据,删除条数: {}", docId, deleteResponse.getData().getDeleteCnt());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void removeByFid(String fid, String kid) {
|
public void removeByFid(String fid, String kid) {
|
||||||
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
|
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
|
||||||
|
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, false);
|
||||||
String expr = "fid == \"" + fid + "\"";
|
Filter filter = MetadataFilterBuilder.metadataKey("fid").isEqualTo(fid);
|
||||||
DeleteParam deleteParam = DeleteParam.newBuilder()
|
embeddingStore.removeAll(filter);
|
||||||
.withCollectionName(collectionName)
|
log.info("Milvus成功删除 fid={} 的所有向量数据", fid);
|
||||||
.withExpr(expr)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<MutationResult> deleteResponse = milvusClient.delete(deleteParam);
|
|
||||||
if (deleteResponse.getStatus() != R.Status.Success.getCode()) {
|
|
||||||
log.error("Milvus删除失败: {}", deleteResponse.getMessage());
|
|
||||||
throw new ServiceException("Milvus删除失败");
|
|
||||||
} else {
|
|
||||||
log.info("Milvus成功删除 fid={} 的所有向量数据,删除条数: {}", fid, deleteResponse.getData().getDeleteCnt());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getVectorStoreType() {
|
||||||
|
return "milvus";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.ruoyi.common.core.config.VectorStoreProperties;
|
import org.ruoyi.common.core.config.VectorStoreProperties;
|
||||||
import org.ruoyi.domain.bo.QueryVectorBo;
|
import org.ruoyi.domain.bo.QueryVectorBo;
|
||||||
import org.ruoyi.domain.bo.StoreEmbeddingBo;
|
import org.ruoyi.domain.bo.StoreEmbeddingBo;
|
||||||
|
import org.ruoyi.embedding.EmbeddingModelFactory;
|
||||||
import org.ruoyi.service.strategy.AbstractVectorStoreStrategy;
|
import org.ruoyi.service.strategy.AbstractVectorStoreStrategy;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
@@ -27,7 +28,7 @@ import java.util.*;
|
|||||||
/**
|
/**
|
||||||
* Weaviate向量库策略实现
|
* Weaviate向量库策略实现
|
||||||
*
|
*
|
||||||
* @author ageer
|
* @author Yzm
|
||||||
*/
|
*/
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
@@ -35,8 +36,8 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
|||||||
|
|
||||||
private WeaviateClient client;
|
private WeaviateClient client;
|
||||||
|
|
||||||
public WeaviateVectorStoreStrategy(VectorStoreProperties vectorStoreProperties) {
|
public WeaviateVectorStoreStrategy(VectorStoreProperties vectorStoreProperties, EmbeddingModelFactory embeddingModelFactory) {
|
||||||
super(vectorStoreProperties);
|
super(vectorStoreProperties, embeddingModelFactory);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -45,7 +46,7 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void createSchema(String vectorModelName, String kid, String modelName) {
|
public void createSchema(String kid, String embeddingModelName) {
|
||||||
String protocol = vectorStoreProperties.getWeaviate().getProtocol();
|
String protocol = vectorStoreProperties.getWeaviate().getProtocol();
|
||||||
String host = vectorStoreProperties.getWeaviate().getHost();
|
String host = vectorStoreProperties.getWeaviate().getHost();
|
||||||
String className = vectorStoreProperties.getWeaviate().getClassname() + kid;
|
String className = vectorStoreProperties.getWeaviate().getClassname() + kid;
|
||||||
@@ -84,9 +85,8 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
|
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
|
||||||
createSchema(storeEmbeddingBo.getVectorModelName(), storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName());
|
createSchema(storeEmbeddingBo.getKid(),storeEmbeddingBo.getEmbeddingModelName());
|
||||||
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(),
|
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), null);
|
||||||
storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl());
|
|
||||||
List<String> chunkList = storeEmbeddingBo.getChunkList();
|
List<String> chunkList = storeEmbeddingBo.getChunkList();
|
||||||
List<String> fidList = storeEmbeddingBo.getFids();
|
List<String> fidList = storeEmbeddingBo.getFids();
|
||||||
String kid = storeEmbeddingBo.getKid();
|
String kid = storeEmbeddingBo.getKid();
|
||||||
@@ -118,9 +118,8 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
|
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
|
||||||
createSchema(queryVectorBo.getVectorModelName(), queryVectorBo.getKid(), queryVectorBo.getVectorModelName());
|
createSchema(queryVectorBo.getKid(),queryVectorBo.getEmbeddingModelName());
|
||||||
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(),
|
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(),null);
|
||||||
queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl());
|
|
||||||
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
|
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
|
||||||
float[] vector = queryEmbedding.vector();
|
float[] vector = queryEmbedding.vector();
|
||||||
List<String> vectorStrings = new ArrayList<>();
|
List<String> vectorStrings = new ArrayList<>();
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
|||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.ruoyi.chain.loader.ResourceLoader;
|
import org.ruoyi.chain.loader.ResourceLoader;
|
||||||
import org.ruoyi.chain.loader.ResourceLoaderFactory;
|
import org.ruoyi.chain.loader.ResourceLoaderFactory;
|
||||||
|
import org.ruoyi.chat.enums.ChatModeType;
|
||||||
import org.ruoyi.common.core.domain.model.LoginUser;
|
import org.ruoyi.common.core.domain.model.LoginUser;
|
||||||
import org.ruoyi.common.core.utils.MapstructUtils;
|
import org.ruoyi.common.core.utils.MapstructUtils;
|
||||||
import org.ruoyi.common.core.utils.StringUtils;
|
import org.ruoyi.common.core.utils.StringUtils;
|
||||||
@@ -237,8 +238,7 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
|
|||||||
}
|
}
|
||||||
baseMapper.insert(knowledgeInfo);
|
baseMapper.insert(knowledgeInfo);
|
||||||
if (knowledgeInfo != null) {
|
if (knowledgeInfo != null) {
|
||||||
vectorStoreService.createSchema(knowledgeInfo.getVectorModelName(),String.valueOf(knowledgeInfo.getId()),
|
vectorStoreService.createSchema(String.valueOf(knowledgeInfo.getId()), bo.getEmbeddingModelName());
|
||||||
bo.getVectorModelName());
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
baseMapper.updateById(knowledgeInfo);
|
baseMapper.updateById(knowledgeInfo);
|
||||||
@@ -313,15 +313,18 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
|
|||||||
.eq(KnowledgeInfo::getId, kid));
|
.eq(KnowledgeInfo::getId, kid));
|
||||||
|
|
||||||
// 通过向量模型查询模型信息
|
// 通过向量模型查询模型信息
|
||||||
ChatModelVo chatModelVo = chatModelService.queryById(knowledgeInfoVo.getEmbeddingModelId());
|
ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModelName());
|
||||||
|
// 未查到指定模型时,回退为向量分类最高优先级模型
|
||||||
|
if (chatModelVo == null) {
|
||||||
|
chatModelVo = chatModelService.selectModelByCategoryWithHighestPriority(ChatModeType.VECTOR.getCode());
|
||||||
|
}
|
||||||
StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo();
|
StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo();
|
||||||
storeEmbeddingBo.setKid(kid);
|
storeEmbeddingBo.setKid(kid);
|
||||||
storeEmbeddingBo.setDocId(docId);
|
storeEmbeddingBo.setDocId(docId);
|
||||||
storeEmbeddingBo.setFids(fids);
|
storeEmbeddingBo.setFids(fids);
|
||||||
storeEmbeddingBo.setChunkList(chunkList);
|
storeEmbeddingBo.setChunkList(chunkList);
|
||||||
storeEmbeddingBo.setVectorModelName(knowledgeInfoVo.getVectorModelName());
|
storeEmbeddingBo.setVectorStoreName(knowledgeInfoVo.getVectorModelName());
|
||||||
storeEmbeddingBo.setEmbeddingModelId(knowledgeInfoVo.getEmbeddingModelId());
|
storeEmbeddingBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModelName());
|
||||||
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
|
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
|
||||||
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
|
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
|
||||||
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
|
vectorStoreService.storeEmbeddings(storeEmbeddingBo);
|
||||||
|
|||||||
Reference in New Issue
Block a user