xieyinlu/ruoyi-ai
1
0
Fork 0
mirror of https://gitcode.com/ageerle/ruoyi-ai.git synced 2026-03-13 20:53:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #226 from Cyclones-Y/main

Browse Source 
完善Milvus向量存储策略实现
This commit is contained in:
ageerle
2025-10-18 13:42:32 +08:00
committed by GitHub
parent 0b1925cc62 e242a67c74
commit 34a71cfc55
16 changed files with 174 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
ruoyi-common/ruoyi-common-core/src/main/java/org/ruoyi/common/core/config/VectorStoreProperties.java
View File

@@ -17,7 +17,7 @@ public class VectorStoreProperties {
/**
* 向量库类型
*/
private String type = "weaviate";
private String type;
/**
* Weaviate配置
@@ -34,17 +34,17 @@ public class VectorStoreProperties {
/**
* 协议
*/
private String protocol = "http";
private String protocol;
/**
* 主机地址
*/
private String host = "localhost:8080";
private String host;
/**
* 类名
*/
private String classname = "Document";
private String classname;
}
@Data
@@ -52,11 +52,11 @@ public class VectorStoreProperties {
/**
* 连接URL
*/
private String url = "http://localhost:19530";
private String url;
/**
* 集合名称
*/
private String collectionname = "knowledge_base";
private String collectionname;
}
}

5
ruoyi-modules-api/ruoyi-chat-api/src/main/java/org/ruoyi/domain/vo/ChatModelVo.java
View File

@@ -70,6 +70,11 @@ public class ChatModelVo implements Serializable {
@ExcelProperty(value = "是否显示")
private String modelShow;
/**
* 模型维度
*/
private Integer dimension;
/**
* 系统提示词
*/

6
ruoyi-modules-api/ruoyi-knowledge-api/pom.xml
View File

@@ -80,6 +80,12 @@
<version>2.6.4</version>
</dependency>
<!-- LangChain4j Milvus Embedding Store -->
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-milvus</artifactId>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-open-ai</artifactId>

4
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/domain/bo/StoreEmbeddingBo.java
View File

@@ -32,9 +32,9 @@ public class StoreEmbeddingBo {
private List<String> fids;
/**
* 向量库模型名称
* 向量库名称
*/
private String vectorModelName;
private String vectorStoreName;
/**
* 向量化模型id

29
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/EmbeddingModelFactory.java
View File

@@ -27,20 +27,23 @@ public class EmbeddingModelFactory {
private final IChatModelService chatModelService;
// 模型缓存，使用ConcurrentHashMap保证线程安全
private final Map<Long, BaseEmbedModelService> modelCache = new ConcurrentHashMap<>();
private final Map<String, BaseEmbedModelService> modelCache = new ConcurrentHashMap<>();
/**
* 创建嵌入模型实例
* 如果模型已存在于缓存中，则直接返回；否则创建新的实例
*
* @param embeddingModelId 嵌入模型的唯一标识ID
* @return BaseEmbedModelService 嵌入模型服务实例
* @param embeddingModelName 嵌入模型名称
* @param dimension 模型维度大小
*/
public BaseEmbedModelService createModel(Long embeddingModelId) {
return modelCache.computeIfAbsent(embeddingModelId, id -> {
ChatModelVo modelConfig = chatModelService.queryById(id);
public BaseEmbedModelService createModel(String embeddingModelName, Integer dimension) {
return modelCache.computeIfAbsent(embeddingModelName, name -> {
ChatModelVo modelConfig = chatModelService.selectModelByName(embeddingModelName);
if (modelConfig == null) {
throw new IllegalArgumentException("未找到模型配置，ID=" + id);
throw new IllegalArgumentException("未找到模型配置，name=" + name);
}
if (modelConfig.getDimension() != null) {
modelConfig.setDimension(dimension);
}
return createModelInstance(modelConfig.getProviderName(), modelConfig);
});
@@ -49,22 +52,22 @@ public class EmbeddingModelFactory {
/**
* 检查模型是否支持多模态
*
* @param embeddingModelId 嵌入模型的唯一标识ID
* @param embeddingModelName 嵌入模型名称
* @return boolean 如果模型支持多模态则返回true，否则返回false
*/
public boolean isMultimodalModel(Long embeddingModelId) {
return createModel(embeddingModelId) instanceof MultiModalEmbedModelService;
public boolean isMultimodalModel(String embeddingModelName) {
return createModel(embeddingModelName, null) instanceof MultiModalEmbedModelService;
}
/**
* 创建多模态嵌入模型实例
*
* @param tenantId 租户ID
* @param embeddingModelName 嵌入模型名称
* @return MultiModalEmbedModelService 多模态嵌入模型服务实例
* @throws IllegalArgumentException 当模型不支持多模态时抛出
*/
public MultiModalEmbedModelService createMultimodalModel(Long tenantId) {
BaseEmbedModelService model = createModel(tenantId);
public MultiModalEmbedModelService createMultimodalModel(String embeddingModelName) {
BaseEmbedModelService model = createModel(embeddingModelName, null);
if (model instanceof MultiModalEmbedModelService) {
return (MultiModalEmbedModelService) model;
}

1
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OllamaEmbeddingProvider.java
View File

@@ -30,6 +30,7 @@ public class OllamaEmbeddingProvider implements BaseEmbedModelService {
return Set.of(ModalityType.TEXT);
}
// ollama不能设置embedding维度，使用milvus时请注意！！创建向量表时需要先设定维度大小
@Override
public Response<List<Embedding>> embedAll(List<TextSegment> textSegments) {
return OllamaEmbeddingModel.builder()

1
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/OpenAiEmbeddingProvider.java
View File

@@ -37,6 +37,7 @@ public class OpenAiEmbeddingProvider implements BaseEmbedModelService {
.baseUrl(chatModelVo.getApiHost())
.apiKey(chatModelVo.getApiKey())
.modelName(chatModelVo.getModelName())
.dimensions(chatModelVo.getDimension())
.build()
.embedAll(textSegments);
}

1
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/embedding/impl/ZhiPuAiEmbeddingProvider.java
View File

@@ -37,6 +37,7 @@ public class ZhiPuAiEmbeddingProvider implements BaseEmbedModelService {
.baseUrl(chatModelVo.getApiHost())
.apiKey(chatModelVo.getApiKey())
.model(chatModelVo.getModelName())
.dimensions(chatModelVo.getDimension())
.build()
.embedAll(textSegments);
}

2
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java
View File

@@ -16,7 +16,7 @@ public interface VectorStoreService {
List<String> getQueryVector(QueryVectorBo queryVectorBo);
void createSchema(String vectorModelName, String kid,String modelName);
void createSchema(String kid, String embeddingModelName);
void removeById(String id,String modelName) throws ServiceException;

22
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java
View File

@@ -2,16 +2,13 @@ package org.ruoyi.service.impl;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.service.ConfigService;
import org.ruoyi.domain.bo.QueryVectorBo;
import org.ruoyi.domain.bo.StoreEmbeddingBo;
import org.ruoyi.service.VectorStoreService;
import org.ruoyi.service.strategy.VectorStoreStrategy;
import org.ruoyi.service.strategy.VectorStoreStrategyFactory;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
/**
* 向量库管理
@@ -30,22 +27,21 @@ public class VectorStoreServiceImpl implements VectorStoreService {
/**
* 获取当前配置的向量库策略
*/
private VectorStoreStrategy getCurrentStrategy() {
private VectorStoreService getCurrentStrategy() {
return strategyFactory.getStrategy();
}
@Override
public void createSchema(String vectorModelName, String kid, String modelName) {
log.info("创建向量库schema: vectorModelName={}, kid={}, modelName={}", vectorModelName, kid, modelName);
VectorStoreStrategy strategy = getCurrentStrategy();
strategy.createSchema(vectorModelName, kid, modelName);
public void createSchema(String kid, String modelName) {
VectorStoreService strategy = getCurrentStrategy();
strategy.createSchema(kid, modelName);
}
@Override
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
log.info("存储向量数据: kid={}, docId={}, 数据条数={}",
storeEmbeddingBo.getKid(), storeEmbeddingBo.getDocId(), storeEmbeddingBo.getChunkList().size());
VectorStoreStrategy strategy = getCurrentStrategy();
VectorStoreService strategy = getCurrentStrategy();
strategy.storeEmbeddings(storeEmbeddingBo);
}
@@ -53,28 +49,28 @@ public class VectorStoreServiceImpl implements VectorStoreService {
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
log.info("查询向量数据: kid={}, query={}, maxResults={}",
queryVectorBo.getKid(), queryVectorBo.getQuery(), queryVectorBo.getMaxResults());
VectorStoreStrategy strategy = getCurrentStrategy();
VectorStoreService strategy = getCurrentStrategy();
return strategy.getQueryVector(queryVectorBo);
}
@Override
public void removeById(String id, String modelName) {
log.info("根据ID删除向量数据: id={}, modelName={}", id, modelName);
VectorStoreStrategy strategy = getCurrentStrategy();
VectorStoreService strategy = getCurrentStrategy();
strategy.removeById(id, modelName);
}
@Override
public void removeByDocId(String docId, String kid) {
log.info("根据docId删除向量数据: docId={}, kid={}", docId, kid);
VectorStoreStrategy strategy = getCurrentStrategy();
VectorStoreService strategy = getCurrentStrategy();
strategy.removeByDocId(docId, kid);
}
@Override
public void removeByFid(String fid, String kid) {
log.info("根据fid删除向量数据: fid={}, kid={}", fid, kid);
VectorStoreStrategy strategy = getCurrentStrategy();
VectorStoreService strategy = getCurrentStrategy();
strategy.removeByFid(fid, kid);
}
}

28
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/AbstractVectorStoreStrategy.java
View File

@@ -8,40 +8,30 @@ import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.config.VectorStoreProperties;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.service.VectorStoreService;
import org.ruoyi.embedding.EmbeddingModelFactory;
/**
* 向量库策略抽象基类
* 提供公共的方法实现，如embedding模型获取等
*
* @author ageer
* @author Yzm
*/
@Slf4j
@RequiredArgsConstructor
public abstract class AbstractVectorStoreStrategy implements VectorStoreStrategy {
public abstract class AbstractVectorStoreStrategy implements VectorStoreService {
protected final VectorStoreProperties vectorStoreProperties;
private final EmbeddingModelFactory embeddingModelFactory;
/**
* 获取向量模型
*/
@SneakyThrows
protected EmbeddingModel getEmbeddingModel(String modelName, String apiKey, String baseUrl) {
EmbeddingModel embeddingModel;
if ("quentinz/bge-large-zh-v1.5".equals(modelName)) {
embeddingModel = OllamaEmbeddingModel.builder()
.baseUrl(baseUrl)
.modelName(modelName)
.build();
} else if ("baai/bge-m3".equals(modelName)) {
embeddingModel = OpenAiEmbeddingModel.builder()
.apiKey(apiKey)
.baseUrl(baseUrl)
.modelName(modelName)
.build();
} else {
throw new ServiceException("未找到对应向量化模型!");
}
return embeddingModel;
protected EmbeddingModel getEmbeddingModel(String modelName, Integer dimension) {
return embeddingModelFactory.createModel(modelName, dimension);
}
/**

18
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategy.java
View File

@@ -1,18 +0,0 @@
package org.ruoyi.service.strategy;
import org.ruoyi.service.VectorStoreService;
/**
* 向量库策略接口
* 继承VectorStoreService以避免重复定义相同的方法
*
* @author ageer
*/
public interface VectorStoreStrategy extends VectorStoreService {
/**
* 获取向量库类型标识
* @return 向量库类型（如：weaviate, milvus）
*/
String getVectorStoreType();
}

27
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategyFactory.java
View File

@@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.config.VectorStoreProperties;
import org.ruoyi.service.strategy.impl.MilvusVectorStoreStrategy;
import org.ruoyi.service.strategy.impl.WeaviateVectorStoreStrategy;
import org.ruoyi.service.VectorStoreService;
import org.springframework.stereotype.Component;
import java.util.HashMap;
@@ -15,7 +16,7 @@ import java.util.Map;
* 向量库策略工厂
* 根据配置动态选择向量库实现
*
* @author ageer
* @author Yzm
*/
@Slf4j
@Component
@@ -26,7 +27,7 @@ public class VectorStoreStrategyFactory {
private final WeaviateVectorStoreStrategy weaviateStrategy;
private final MilvusVectorStoreStrategy milvusStrategy;
private Map<String, VectorStoreStrategy> strategies;
private Map<String, VectorStoreService> strategies;
@PostConstruct
public void init() {
@@ -39,36 +40,18 @@ public class VectorStoreStrategyFactory {
/**
* 获取当前配置的向量库策略
*/
public VectorStoreStrategy getStrategy() {
public VectorStoreService getStrategy() {
String vectorStoreType = vectorStoreProperties.getType();
if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) {
vectorStoreType = "weaviate"; // 默认使用weaviate
}
VectorStoreStrategy strategy = strategies.get(vectorStoreType.toLowerCase());
VectorStoreService strategy = strategies.get(vectorStoreType.toLowerCase());
if (strategy == null) {
log.warn("未找到向量库策略: {}, 使用默认策略: weaviate", vectorStoreType);
strategy = strategies.get("weaviate");
}
log.debug("使用向量库策略: {}", vectorStoreType);
return strategy;
}
/**
* 根据类型获取向量库策略
*/
public VectorStoreStrategy getStrategy(String type) {
if (type == null || type.trim().isEmpty()) {
return getStrategy();
}
VectorStoreStrategy strategy = strategies.get(type.toLowerCase());
if (strategy == null) {
log.warn("未找到向量库策略: {}, 使用默认策略", type);
return getStrategy();
}
return strategy;
}
}

368
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java
View File

@@ -1,337 +1,157 @@
package org.ruoyi.service.strategy.impl;
import org.ruoyi.common.core.exception.ServiceException;
import dev.langchain4j.data.document.Metadata;
import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.embedding.EmbeddingModel;
import io.milvus.client.MilvusServiceClient;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
import io.milvus.grpc.*;
import io.milvus.param.*;
import io.milvus.param.collection.*;
import io.milvus.param.dml.DeleteParam;
import io.milvus.param.dml.InsertParam;
import io.milvus.param.dml.SearchParam;
import io.milvus.param.index.CreateIndexParam;
import io.milvus.param.index.DescribeIndexParam;
import io.milvus.response.DescCollResponseWrapper;
import io.milvus.response.SearchResultsWrapper;
import dev.langchain4j.store.embedding.EmbeddingMatch;
import dev.langchain4j.store.embedding.EmbeddingSearchRequest;
import dev.langchain4j.store.embedding.EmbeddingStore;
import dev.langchain4j.store.embedding.filter.Filter;
import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder;
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
import io.milvus.param.IndexType;
import io.milvus.param.MetricType;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.config.VectorStoreProperties;
import org.ruoyi.domain.bo.QueryVectorBo;
import org.ruoyi.domain.bo.StoreEmbeddingBo;
import org.ruoyi.embedding.EmbeddingModelFactory;
import org.ruoyi.service.strategy.AbstractVectorStoreStrategy;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.IntStream;
/**
* Milvus向量库策略实现
*
* @author ageer
*/
@Slf4j
@Component
public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy {
private MilvusServiceClient milvusClient;
public MilvusVectorStoreStrategy(VectorStoreProperties vectorStoreProperties) {
super(vectorStoreProperties);
private final Integer DIMENSION = 2048;
public MilvusVectorStoreStrategy(VectorStoreProperties vectorStoreProperties, EmbeddingModelFactory embeddingModelFactory) {
super(vectorStoreProperties, embeddingModelFactory);
}
// 缓存不同集合与 autoFlush 配置的 Milvus 连接
private final Map<String, EmbeddingStore<TextSegment>> storeCache = new ConcurrentHashMap<>();
private EmbeddingStore<TextSegment> getMilvusStore(String collectionName, boolean autoFlushOnInsert) {
String key = collectionName + "|" + autoFlushOnInsert;
return storeCache.computeIfAbsent(key, k ->
MilvusEmbeddingStore.builder()
.uri(vectorStoreProperties.getMilvus().getUrl())
.collectionName(collectionName)
.dimension(DIMENSION)
.indexType(IndexType.IVF_FLAT)
.metricType(MetricType.L2)
.autoFlushOnInsert(autoFlushOnInsert)
.idFieldName("id")
.textFieldName("text")
.metadataFieldName("metadata")
.vectorFieldName("vector")
.build()
);
}
@Override
public String getVectorStoreType() {
return "milvus";
}
@Override
public void createSchema(String vectorModelName, String kid, String modelName) {
String url = vectorStoreProperties.getMilvus().getUrl();
public void createSchema(String kid, String modelName) {
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
// 创建Milvus客户端连接
ConnectParam connectParam = ConnectParam.newBuilder()
.withUri(url)
.build();
milvusClient = new MilvusServiceClient(connectParam);
// 检查集合是否存在
HasCollectionParam hasCollectionParam = HasCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build();
R<Boolean> hasCollectionResponse = milvusClient.hasCollection(hasCollectionParam);
if (hasCollectionResponse.getStatus() != R.Status.Success.getCode()) {
log.error("检查集合是否存在失败: {}", hasCollectionResponse.getMessage());
return;
}
if (!hasCollectionResponse.getData()) {
// 创建字段
List<FieldType> fields = new ArrayList<>();
// ID字段 (主键)
fields.add(FieldType.newBuilder()
.withName("id")
.withDataType(DataType.Int64)
.withPrimaryKey(true)
.withAutoID(true)
.build());
// 文本字段
fields.add(FieldType.newBuilder()
.withName("text")
.withDataType(DataType.VarChar)
.withMaxLength(65535)
.build());
// fid字段
fields.add(FieldType.newBuilder()
.withName("fid")
.withDataType(DataType.VarChar)
.withMaxLength(255)
.build());
// kid字段
fields.add(FieldType.newBuilder()
.withName("kid")
.withDataType(DataType.VarChar)
.withMaxLength(255)
.build());
// docId字段
fields.add(FieldType.newBuilder()
.withName("docId")
.withDataType(DataType.VarChar)
.withMaxLength(255)
.build());
// 向量字段
fields.add(FieldType.newBuilder()
.withName("vector")
.withDataType(DataType.FloatVector)
.withDimension(1024) // 根据实际embedding维度调整
.build());
// 创建集合
CreateCollectionParam createCollectionParam = CreateCollectionParam.newBuilder()
.withCollectionName(collectionName)
.withDescription("Knowledge base collection for " + kid)
.withShardsNum(2)
.withFieldTypes(fields)
.build();
R<RpcStatus> createCollectionResponse = milvusClient.createCollection(createCollectionParam);
if (createCollectionResponse.getStatus() != R.Status.Success.getCode()) {
log.error("创建集合失败: {}", createCollectionResponse.getMessage());
return;
}
// 创建索引
CreateIndexParam createIndexParam = CreateIndexParam.newBuilder()
.withCollectionName(collectionName)
.withFieldName("vector")
.withIndexType(IndexType.IVF_FLAT)
.withMetricType(MetricType.L2)
.withExtraParam("{\"nlist\":1024}")
.build();
R<RpcStatus> createIndexResponse = milvusClient.createIndex(createIndexParam);
if (createIndexResponse.getStatus() != R.Status.Success.getCode()) {
log.error("创建索引失败: {}", createIndexResponse.getMessage());
} else {
log.info("Milvus集合和索引创建成功: {}", collectionName);
}
} else {
log.info("Milvus集合已存在: {}", collectionName);
}
// 使用缓存获取连接以确保只初始化一次
EmbeddingStore<TextSegment> store = getMilvusStore(collectionName, true);
log.info("Milvus集合初始化完成: {}", collectionName);
}
@Override
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
createSchema(storeEmbeddingBo.getVectorModelName(), storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName());
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(),
storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl());
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), DIMENSION);
List<String> chunkList = storeEmbeddingBo.getChunkList();
List<String> fidList = storeEmbeddingBo.getFids();
String kid = storeEmbeddingBo.getKid();
String docId = storeEmbeddingBo.getDocId();
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
log.info("Milvus向量存储条数记录: " + chunkList.size());
log.info("Milvus向量存储条数记录: {}", chunkList.size());
long startTime = System.currentTimeMillis();
// 准备批量插入数据
List<InsertParam.Field> fields = new ArrayList<>();
List<String> textList = new ArrayList<>();
List<String> fidListData = new ArrayList<>();
List<String> kidList = new ArrayList<>();
List<String> docIdList = new ArrayList<>();
List<List<Float>> vectorList = new ArrayList<>();
for (int i = 0; i < chunkList.size(); i++) {
// 复用连接，写入场景使用 autoFlush=false 以提升批量插入性能
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, false);
IntStream.range(0, chunkList.size()).forEach(i -> {
String text = chunkList.get(i);
String fid = fidList.get(i);
Metadata metadata = new Metadata();
metadata.put("fid", fid);
metadata.put("kid", kid);
metadata.put("docId", docId);
TextSegment textSegment = TextSegment.from(text, metadata);
Embedding embedding = embeddingModel.embed(text).content();
textList.add(text);
fidListData.add(fid);
kidList.add(kid);
docIdList.add(docId);
List<Float> vector = new ArrayList<>();
for (float f : embedding.vector()) {
vector.add(f);
}
vectorList.add(vector);
}
// 构建字段数据
fields.add(new InsertParam.Field("text", textList));
fields.add(new InsertParam.Field("fid", fidListData));
fields.add(new InsertParam.Field("kid", kidList));
fields.add(new InsertParam.Field("docId", docIdList));
fields.add(new InsertParam.Field("vector", vectorList));
// 执行插入
InsertParam insertParam = InsertParam.newBuilder()
.withCollectionName(collectionName)
.withFields(fields)
.build();
R<MutationResult> insertResponse = milvusClient.insert(insertParam);
if (insertResponse.getStatus() != R.Status.Success.getCode()) {
log.error("Milvus向量存储失败: {}", insertResponse.getMessage());
throw new ServiceException("Milvus向量存储失败");
} else {
log.info("Milvus向量存储成功，插入条数: {}", insertResponse.getData().getInsertCnt());
}
embeddingStore.add(embedding, textSegment);
});
long endTime = System.currentTimeMillis();
log.info("Milvus向量存储完成消耗时间：" + (endTime - startTime) / 1000 + "秒");
log.info("Milvus向量存储完成消耗时间：{}秒", (endTime - startTime) / 1000);
}
@Override
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
createSchema(queryVectorBo.getVectorModelName(), queryVectorBo.getKid(), queryVectorBo.getVectorModelName());
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(),
queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl());
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(), DIMENSION);
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + queryVectorBo.getKid();
// 查询复用连接，autoFlush 对查询无影响，此处保持 true
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, true);
List<String> resultList = new ArrayList<>();
// 加载集合到内存
LoadCollectionParam loadCollectionParam = LoadCollectionParam.newBuilder()
.withCollectionName(collectionName)
EmbeddingSearchRequest request = EmbeddingSearchRequest.builder()
.queryEmbedding(queryEmbedding)
.maxResults(queryVectorBo.getMaxResults())
.build();
milvusClient.loadCollection(loadCollectionParam);
// 准备查询向量
List<List<Float>> searchVectors = new ArrayList<>();
List<Float> queryVector = new ArrayList<>();
for (float f : queryEmbedding.vector()) {
queryVector.add(f);
}
searchVectors.add(queryVector);
// 构建搜索参数
SearchParam searchParam = SearchParam.newBuilder()
.withCollectionName(collectionName)
.withMetricType(MetricType.L2)
.withOutFields(Arrays.asList("text", "fid", "kid", "docId"))
.withTopK(queryVectorBo.getMaxResults())
.withVectors(searchVectors)
.withVectorFieldName("vector")
.withParams("{\"nprobe\":10}")
.build();
R<SearchResults> searchResponse = milvusClient.search(searchParam);
if (searchResponse.getStatus() != R.Status.Success.getCode()) {
log.error("Milvus查询失败: {}", searchResponse.getMessage());
return resultList;
}
SearchResultsWrapper wrapper = new SearchResultsWrapper(searchResponse.getData().getResults());
// 遍历搜索结果
for (int i = 0; i < wrapper.getIDScore(0).size(); i++) {
SearchResultsWrapper.IDScore idScore = wrapper.getIDScore(0).get(i);
// 获取text字段数据
List<?> textFieldData = wrapper.getFieldData("text", 0);
if (textFieldData != null && i < textFieldData.size()) {
Object textObj = textFieldData.get(i);
if (textObj != null) {
resultList.add(textObj.toString());
log.debug("找到相似文本，ID: {}, 距离: {}, 内容: {}",
idScore.getLongID(), idScore.getScore(), textObj.toString());
}
List<EmbeddingMatch<TextSegment>> matches = embeddingStore.search(request).matches();
for (EmbeddingMatch<TextSegment> match : matches) {
TextSegment segment = match.embedded();
if (segment != null) {
resultList.add(segment.text());
}
}
return resultList;
}
@Override
@SneakyThrows
public void removeById(String id, String modelName) {
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + id;
// 删除整个集合
DropCollectionParam dropCollectionParam = DropCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build();
R<RpcStatus> dropResponse = milvusClient.dropCollection(dropCollectionParam);
if (dropResponse.getStatus() != R.Status.Success.getCode()) {
log.error("Milvus集合删除失败: {}", dropResponse.getMessage());
throw new ServiceException("Milvus集合删除失败");
} else {
log.info("Milvus集合删除成功: {}", collectionName);
}
// 注意：此处原逻辑使用 collectionname + id，保持现状
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(vectorStoreProperties.getMilvus().getCollectionname() + id, false);
embeddingStore.remove(id);
}
@Override
public void removeByDocId(String docId, String kid) {
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
String expr = "docId == \"" + docId + "\"";
DeleteParam deleteParam = DeleteParam.newBuilder()
.withCollectionName(collectionName)
.withExpr(expr)
.build();
R<MutationResult> deleteResponse = milvusClient.delete(deleteParam);
if (deleteResponse.getStatus() != R.Status.Success.getCode()) {
log.error("Milvus删除失败: {}", deleteResponse.getMessage());
throw new ServiceException("Milvus删除失败");
} else {
log.info("Milvus成功删除 docId={} 的所有向量数据，删除条数: {}", docId, deleteResponse.getData().getDeleteCnt());
}
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, false);
Filter filter = MetadataFilterBuilder.metadataKey("docId").isEqualTo(docId);
embeddingStore.removeAll(filter);
log.info("Milvus成功删除 docId={} 的所有向量数据", docId);
}
@Override
public void removeByFid(String fid, String kid) {
String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid;
String expr = "fid == \"" + fid + "\"";
DeleteParam deleteParam = DeleteParam.newBuilder()
.withCollectionName(collectionName)
.withExpr(expr)
.build();
R<MutationResult> deleteResponse = milvusClient.delete(deleteParam);
if (deleteResponse.getStatus() != R.Status.Success.getCode()) {
log.error("Milvus删除失败: {}", deleteResponse.getMessage());
throw new ServiceException("Milvus删除失败");
} else {
log.info("Milvus成功删除 fid={} 的所有向量数据，删除条数: {}", fid, deleteResponse.getData().getDeleteCnt());
}
EmbeddingStore<TextSegment> embeddingStore = getMilvusStore(collectionName, false);
Filter filter = MetadataFilterBuilder.metadataKey("fid").isEqualTo(fid);
embeddingStore.removeAll(filter);
log.info("Milvus成功删除 fid={} 的所有向量数据", fid);
}
}
@Override
public String getVectorStoreType() {
return "milvus";
}
}

19
ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/WeaviateVectorStoreStrategy.java
View File

@@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.config.VectorStoreProperties;
import org.ruoyi.domain.bo.QueryVectorBo;
import org.ruoyi.domain.bo.StoreEmbeddingBo;
import org.ruoyi.embedding.EmbeddingModelFactory;
import org.ruoyi.service.strategy.AbstractVectorStoreStrategy;
import org.springframework.stereotype.Component;
import java.util.*;
@@ -27,7 +28,7 @@ import java.util.*;
/**
* Weaviate向量库策略实现
*
* @author ageer
* @author Yzm
*/
@Slf4j
@Component
@@ -35,8 +36,8 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy {
private WeaviateClient client;
public WeaviateVectorStoreStrategy(VectorStoreProperties vectorStoreProperties) {
super(vectorStoreProperties);
public WeaviateVectorStoreStrategy(VectorStoreProperties vectorStoreProperties, EmbeddingModelFactory embeddingModelFactory) {
super(vectorStoreProperties, embeddingModelFactory);
}
@Override
@@ -45,7 +46,7 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy {
}
@Override
public void createSchema(String vectorModelName, String kid, String modelName) {
public void createSchema(String kid, String embeddingModelName) {
String protocol = vectorStoreProperties.getWeaviate().getProtocol();
String host = vectorStoreProperties.getWeaviate().getHost();
String className = vectorStoreProperties.getWeaviate().getClassname() + kid;
@@ -84,9 +85,8 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy {
@Override
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
createSchema(storeEmbeddingBo.getVectorModelName(), storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName());
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(),
storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl());
createSchema(storeEmbeddingBo.getKid(),storeEmbeddingBo.getEmbeddingModelName());
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), null);
List<String> chunkList = storeEmbeddingBo.getChunkList();
List<String> fidList = storeEmbeddingBo.getFids();
String kid = storeEmbeddingBo.getKid();
@@ -118,9 +118,8 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy {
@Override
public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
createSchema(queryVectorBo.getVectorModelName(), queryVectorBo.getKid(), queryVectorBo.getVectorModelName());
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(),
queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl());
createSchema(queryVectorBo.getKid(),queryVectorBo.getEmbeddingModelName());
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(),null);
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
float[] vector = queryEmbedding.vector();
List<String> vectorStrings = new ArrayList<>();

15
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/KnowledgeInfoServiceImpl.java
View File

@@ -9,6 +9,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.RequiredArgsConstructor;
import org.ruoyi.chain.loader.ResourceLoader;
import org.ruoyi.chain.loader.ResourceLoaderFactory;
import org.ruoyi.chat.enums.ChatModeType;
import org.ruoyi.common.core.domain.model.LoginUser;
import org.ruoyi.common.core.utils.MapstructUtils;
import org.ruoyi.common.core.utils.StringUtils;
@@ -237,8 +238,7 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
}
baseMapper.insert(knowledgeInfo);
if (knowledgeInfo != null) {
vectorStoreService.createSchema(knowledgeInfo.getVectorModelName(),String.valueOf(knowledgeInfo.getId()),
bo.getVectorModelName());
vectorStoreService.createSchema(String.valueOf(knowledgeInfo.getId()), bo.getEmbeddingModelName());
}
} else {
baseMapper.updateById(knowledgeInfo);
@@ -313,15 +313,18 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
.eq(KnowledgeInfo::getId, kid));
// 通过向量模型查询模型信息
ChatModelVo chatModelVo = chatModelService.queryById(knowledgeInfoVo.getEmbeddingModelId());
ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModelName());
// 未查到指定模型时，回退为向量分类最高优先级模型
if (chatModelVo == null) {
chatModelVo = chatModelService.selectModelByCategoryWithHighestPriority(ChatModeType.VECTOR.getCode());
}
StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo();
storeEmbeddingBo.setKid(kid);
storeEmbeddingBo.setDocId(docId);
storeEmbeddingBo.setFids(fids);
storeEmbeddingBo.setChunkList(chunkList);
storeEmbeddingBo.setVectorModelName(knowledgeInfoVo.getVectorModelName());
storeEmbeddingBo.setEmbeddingModelId(knowledgeInfoVo.getEmbeddingModelId());
storeEmbeddingBo.setVectorStoreName(knowledgeInfoVo.getVectorModelName());
storeEmbeddingBo.setEmbeddingModelName(knowledgeInfoVo.getEmbeddingModelName());
storeEmbeddingBo.setApiKey(chatModelVo.getApiKey());
storeEmbeddingBo.setBaseUrl(chatModelVo.getApiHost());
vectorStoreService.storeEmbeddings(storeEmbeddingBo);