From aa1c771e726e657604619c3db6592a0232a8565f Mon Sep 17 00:00:00 2001 From: Yzm Date: Thu, 25 Sep 2025 18:44:19 +0800 Subject: [PATCH 1/6] =?UTF-8?q?feat(knowledge):=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E5=90=91=E9=87=8F=E5=BA=93=E7=AD=96=E7=95=A5=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=9A=E5=90=91=E9=87=8F=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增向量库策略接口及抽象基类 - 实现Weaviate向量库策略- 实现Milvus向量库策略(占位实现) - 添加向量库策略工厂类动态选择实现 - 修改向量存储服务使用策略模式 - 更新知识信息service调用参数顺序 - 添加文档分段和知识片段ID生成注释 - 修改dev环境数据库配置为github版本 --- .../org/ruoyi/service/VectorStoreService.java | 2 +- .../service/impl/VectorStoreServiceImpl.java | 264 +++------------ .../strategy/AbstractVectorStoreStrategy.java | 62 ++++ .../service/strategy/VectorStoreStrategy.java | 18 + .../strategy/VectorStoreStrategyFactory.java | 88 +++++ .../impl/MilvusVectorStoreStrategy.java | 312 ++++++++++++++++++ .../impl/WeaviateVectorStoreStrategy.java | 233 +++++++++++++ .../knowledge/KnowledgeInfoServiceImpl.java | 4 +- 8 files changed, 754 insertions(+), 229 deletions(-) create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/AbstractVectorStoreStrategy.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategy.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategyFactory.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java create mode 100644 ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/WeaviateVectorStoreStrategy.java diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java index 29bfecc3..21d2410d 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java @@ -15,7 +15,7 @@ public interface VectorStoreService { List getQueryVector(QueryVectorBo queryVectorBo); - void createSchema(String kid,String modelName); + void createSchema(String vectorModelName, String kid,String modelName); void removeById(String id,String modelName); diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java index 799ce729..e7b023cc 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java @@ -1,40 +1,19 @@ package org.ruoyi.service.impl; -import cn.hutool.json.JSONObject; -import com.google.protobuf.ServiceException; -import dev.langchain4j.data.embedding.Embedding; -import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.model.ollama.OllamaEmbeddingModel; -import dev.langchain4j.model.openai.OpenAiEmbeddingModel; -import dev.langchain4j.store.embedding.EmbeddingMatch; -import dev.langchain4j.store.embedding.EmbeddingSearchRequest; -import dev.langchain4j.store.embedding.EmbeddingStore; -import dev.langchain4j.store.embedding.weaviate.WeaviateEmbeddingStore; -import io.weaviate.client.Config; -import io.weaviate.client.WeaviateClient; -import io.weaviate.client.base.Result; -import io.weaviate.client.v1.batch.api.ObjectsBatchDeleter; -import io.weaviate.client.v1.batch.model.BatchDeleteResponse; -import io.weaviate.client.v1.filters.Operator; -import io.weaviate.client.v1.filters.WhereFilter; -import io.weaviate.client.v1.graphql.model.GraphQLResponse; -import io.weaviate.client.v1.schema.model.Property; -import io.weaviate.client.v1.schema.model.Schema; -import io.weaviate.client.v1.schema.model.WeaviateClass; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.ruoyi.common.core.service.ConfigService; import org.ruoyi.domain.bo.QueryVectorBo; import org.ruoyi.domain.bo.StoreEmbeddingBo; import org.ruoyi.service.VectorStoreService; +import org.ruoyi.service.strategy.VectorStoreStrategy; +import org.ruoyi.service.strategy.VectorStoreStrategyFactory; import org.springframework.stereotype.Service; -import java.util.*; -import java.util.stream.Collectors; + +import java.util.List; /** - * 向量库管理 + * 向量库管理服务实现 - 使用策略模式 * * @author ageer */ @@ -44,230 +23,61 @@ import java.util.stream.Collectors; public class VectorStoreServiceImpl implements VectorStoreService { private final ConfigService configService; + private final VectorStoreStrategyFactory strategyFactory; -// private EmbeddingStore embeddingStore; - private WeaviateClient client; + /** + * 获取当前配置的向量库策略 + */ + private VectorStoreStrategy getCurrentStrategy() { + String vectorStoreType = configService.getConfigValue("vector", "type"); + if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) { + vectorStoreType = "weaviate"; // 默认使用weaviate + } + return strategyFactory.getStrategy(vectorStoreType); + } @Override - public void createSchema(String kid, String modelName) { - String protocol = configService.getConfigValue("weaviate", "protocol"); - String host = configService.getConfigValue("weaviate", "host"); - String className = configService.getConfigValue("weaviate", "classname")+kid; - // 创建 Weaviate 客户端 - client= new WeaviateClient(new Config(protocol, host)); - // 检查类是否存在，如果不存在就创建 schema - Result schemaResult = client.schema().getter().run(); - Schema schema = schemaResult.getResult(); - boolean classExists = false; - for (WeaviateClass weaviateClass : schema.getClasses()) { - if (weaviateClass.getClassName().equals(className)) { - classExists = true; - break; - } - } - if (!classExists) { - // 类不存在，创建 schema - WeaviateClass build = WeaviateClass.builder() - .className(className) - .vectorizer("none") - .properties( - List.of(Property.builder().name("text").dataType(Collections.singletonList("text")).build(), - Property.builder().name("fid").dataType(Collections.singletonList("text")).build(), - Property.builder().name("kid").dataType(Collections.singletonList("text")).build(), - Property.builder().name("docId").dataType(Collections.singletonList("text")).build()) - ) - .build(); - Result createResult = client.schema().classCreator().withClass(build).run(); - if (createResult.hasErrors()) { - log.error("Schema 创建失败: {}", createResult.getError()); - } else { - log.info("Schema 创建成功: {}", className); - } - } -// embeddingStore = WeaviateEmbeddingStore.builder() -// .scheme(protocol) -// .host(host) -// .objectClass(className) -// .scheme(protocol) -// .avoidDups(true) -// .consistencyLevel("ALL") -// .build(); + public void createSchema(String vectorModelName, String kid, String modelName) { + log.info("创建向量库schema: vectorModelName={}, kid={}, modelName={}", vectorModelName, kid, modelName); + VectorStoreStrategy strategy = getCurrentStrategy(); + strategy.createSchema(vectorModelName, kid, modelName); } @Override public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) { - createSchema(storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName()); - EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), - storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl()); - List chunkList = storeEmbeddingBo.getChunkList(); - List fidList = storeEmbeddingBo.getFids(); - String kid = storeEmbeddingBo.getKid(); - String docId = storeEmbeddingBo.getDocId(); - log.info("向量存储条数记录: " + chunkList.size()); - long startTime = System.currentTimeMillis(); - for (int i = 0; i < chunkList.size(); i++) { - String text = chunkList.get(i); - String fid = fidList.get(i); - Embedding embedding = embeddingModel.embed(text).content(); - Map properties = Map.of( - "text", text, - "fid",fid, - "kid", kid, - "docId", docId - ); - Float[] vector = toObjectArray(embedding.vector()); - client.data().creator() - .withClassName("LocalKnowledge" + kid) // 注意替换成实际类名 - .withProperties(properties) - .withVector(vector) - .run(); - } - long endTime = System.currentTimeMillis(); - log.info("向量存储完成消耗时间："+ (endTime-startTime)/1000+"秒"); + log.info("存储向量数据: kid={}, docId={}, 数据条数={}", + storeEmbeddingBo.getKid(), storeEmbeddingBo.getDocId(), storeEmbeddingBo.getChunkList().size()); + VectorStoreStrategy strategy = getCurrentStrategy(); + strategy.storeEmbeddings(storeEmbeddingBo); } - private static Float[] toObjectArray(float[] primitive) { - Float[] result = new Float[primitive.length]; - for (int i = 0; i < primitive.length; i++) { - result[i] = primitive[i]; // 自动装箱 - } - return result; - } @Override public List getQueryVector(QueryVectorBo queryVectorBo) { - createSchema(queryVectorBo.getKid(), queryVectorBo.getVectorModelName()); - EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(), - queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl()); - Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content(); - float[] vector = queryEmbedding.vector(); - List vectorStrings = new ArrayList<>(); - for (float v : vector) { - vectorStrings.add(String.valueOf(v)); - } - String vectorStr = String.join(",", vectorStrings); - String className = configService.getConfigValue("weaviate", "classname") ; - // 构建 GraphQL 查询 - String graphQLQuery = String.format( - "{\n" + - " Get {\n" + - " %s(nearVector: {vector: [%s]} limit: %d) {\n" + - " text\n" + - " fid\n" + - " kid\n" + - " docId\n" + - " _additional {\n" + - " distance\n" + - " id\n" + - " }\n" + - " }\n" + - " }\n" + - "}", - className+ queryVectorBo.getKid(), - vectorStr, - queryVectorBo.getMaxResults() - ); - - Result result = client.graphQL().raw().withQuery(graphQLQuery).run(); - List resultList = new ArrayList<>(); - if (result != null && !result.hasErrors()) { - Object data = result.getResult().getData(); - JSONObject entries = new JSONObject(data); - Map entriesMap = entries.get("Get", Map.class); - cn.hutool.json.JSONArray objects = entriesMap.get(className + queryVectorBo.getKid()); - if(objects.isEmpty()){ - return resultList; - } - for (Object object : objects) { - Map map = (Map) object; - String content = map.get("text"); - resultList.add( content); - } - return resultList; - } else { - log.error("GraphQL 查询失败: {}", result.getError()); - return resultList; - } + log.info("查询向量数据: kid={}, query={}, maxResults={}", + queryVectorBo.getKid(), queryVectorBo.getQuery(), queryVectorBo.getMaxResults()); + VectorStoreStrategy strategy = getCurrentStrategy(); + return strategy.getQueryVector(queryVectorBo); } @Override - @SneakyThrows public void removeById(String id, String modelName) { - String protocol = configService.getConfigValue("weaviate", "protocol"); - String host = configService.getConfigValue("weaviate", "host"); - String className = configService.getConfigValue("weaviate", "classname"); - String finalClassName = className + id; - WeaviateClient client = new WeaviateClient(new Config(protocol, host)); - Result result = client.schema().classDeleter().withClassName(finalClassName).run(); - if (result.hasErrors()) { - log.error("失败删除向量: " + result.getError()); - throw new ServiceException("失败删除向量数据!"); - } else { - log.info("成功删除向量数据: " + result.getResult()); - } + log.info("根据ID删除向量数据: id={}, modelName={}", id, modelName); + VectorStoreStrategy strategy = getCurrentStrategy(); + strategy.removeById(id, modelName); } @Override public void removeByDocId(String docId, String kid) { - String className = configService.getConfigValue("weaviate", "classname") + kid; - // 构建 Where 条件 - WhereFilter whereFilter = WhereFilter.builder() - .path("docId") - .operator(Operator.Equal) - .valueText(docId) - .build(); - ObjectsBatchDeleter deleter = client.batch().objectsBatchDeleter(); - Result result = deleter.withClassName(className) - .withWhere(whereFilter) - .run(); - if (result != null && !result.hasErrors()) { - log.info("成功删除 docId={} 的所有向量数据", docId); - } else { - log.error("删除失败: {}", result.getError()); - } + log.info("根据docId删除向量数据: docId={}, kid={}", docId, kid); + VectorStoreStrategy strategy = getCurrentStrategy(); + strategy.removeByDocId(docId, kid); } @Override public void removeByFid(String fid, String kid) { - String className = configService.getConfigValue("weaviate", "classname") + kid; - // 构建 Where 条件 - WhereFilter whereFilter = WhereFilter.builder() - .path("fid") - .operator(Operator.Equal) - .valueText(fid) - .build(); - ObjectsBatchDeleter deleter = client.batch().objectsBatchDeleter(); - Result result = deleter.withClassName(className) - .withWhere(whereFilter) - .run(); - if (result != null && !result.hasErrors()) { - log.info("成功删除 fid={} 的所有向量数据", fid); - } else { - log.error("删除失败: {}", result.getError()); - } + log.info("根据fid删除向量数据: fid={}, kid={}", fid, kid); + VectorStoreStrategy strategy = getCurrentStrategy(); + strategy.removeByFid(fid, kid); } - - /** - * 获取向量模型 - */ - @SneakyThrows - public EmbeddingModel getEmbeddingModel(String modelName, String apiKey, String baseUrl) { - EmbeddingModel embeddingModel; - if ("quentinz/bge-large-zh-v1.5".equals(modelName)) { - embeddingModel = OllamaEmbeddingModel.builder() - .baseUrl(baseUrl) - .modelName(modelName) - .build(); - } else if ("baai/bge-m3".equals(modelName)) { - embeddingModel = OpenAiEmbeddingModel.builder() - .apiKey(apiKey) - .baseUrl(baseUrl) - .modelName(modelName) - .build(); - } else { - throw new ServiceException("未找到对应向量化模型!"); - } - return embeddingModel; - } - } diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/AbstractVectorStoreStrategy.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/AbstractVectorStoreStrategy.java new file mode 100644 index 00000000..104714cb --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/AbstractVectorStoreStrategy.java @@ -0,0 +1,62 @@ +package org.ruoyi.service.strategy; + +import com.google.protobuf.ServiceException; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.ollama.OllamaEmbeddingModel; +import dev.langchain4j.model.openai.OpenAiEmbeddingModel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.common.core.service.ConfigService; + +/** + * 向量库策略抽象基类 + * 提供公共的方法实现，如embedding模型获取等 + * + * @author ageer + */ +@Slf4j +@RequiredArgsConstructor +public abstract class AbstractVectorStoreStrategy implements VectorStoreStrategy { + + protected final ConfigService configService; + + /** + * 获取向量模型 + */ + @SneakyThrows + protected EmbeddingModel getEmbeddingModel(String modelName, String apiKey, String baseUrl) { + EmbeddingModel embeddingModel; + if ("quentinz/bge-large-zh-v1.5".equals(modelName)) { + embeddingModel = OllamaEmbeddingModel.builder() + .baseUrl(baseUrl) + .modelName(modelName) + .build(); + } else if ("baai/bge-m3".equals(modelName)) { + embeddingModel = OpenAiEmbeddingModel.builder() + .apiKey(apiKey) + .baseUrl(baseUrl) + .modelName(modelName) + .build(); + } else { + throw new ServiceException("未找到对应向量化模型!"); + } + return embeddingModel; + } + + /** + * 将float数组转换为Float对象数组 + */ + protected static Float[] toObjectArray(float[] primitive) { + Float[] result = new Float[primitive.length]; + for (int i = 0; i < primitive.length; i++) { + result[i] = primitive[i]; // 自动装箱 + } + return result; + } + + /** + * 获取向量库类型标识 + */ + public abstract String getVectorStoreType(); +} \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategy.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategy.java new file mode 100644 index 00000000..bd93e6fa --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategy.java @@ -0,0 +1,18 @@ +package org.ruoyi.service.strategy; + +import org.ruoyi.service.VectorStoreService; + +/** + * 向量库策略接口 + * 继承VectorStoreService以避免重复定义相同的方法 + * + * @author ageer + */ +public interface VectorStoreStrategy extends VectorStoreService { + + /** + * 获取向量库类型标识 + * @return 向量库类型（如：weaviate, milvus） + */ + String getVectorStoreType(); +} \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategyFactory.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategyFactory.java new file mode 100644 index 00000000..1e606b22 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategyFactory.java @@ -0,0 +1,88 @@ +package org.ruoyi.service.strategy; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.common.core.service.ConfigService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 向量库策略工厂 + * 根据配置动态选择向量库实现 + * + * @author ageer + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class VectorStoreStrategyFactory implements ApplicationContextAware { + + private final ConfigService configService; + private final Map strategyMap = new ConcurrentHashMap<>(); + private ApplicationContext applicationContext; + + @Override + public void setApplicationContext(ApplicationContext applicationContext) { + this.applicationContext = applicationContext; + initStrategies(); + } + + /** + * 初始化所有策略实现 + */ + private void initStrategies() { + Map strategies = applicationContext.getBeansOfType(VectorStoreStrategy.class); + for (VectorStoreStrategy strategy : strategies.values()) { + if (strategy instanceof AbstractVectorStoreStrategy) { + AbstractVectorStoreStrategy abstractStrategy = (AbstractVectorStoreStrategy) strategy; + strategyMap.put(abstractStrategy.getVectorStoreType(), strategy); + log.info("注册向量库策略: {}", abstractStrategy.getVectorStoreType()); + } + } + } + + /** + * 获取当前配置的向量库策略 + */ + public VectorStoreStrategy getStrategy() { + String vectorStoreType = configService.getConfigValue("vector", "store_type"); + if (vectorStoreType == null || vectorStoreType.isEmpty()) { + vectorStoreType = "weaviate"; // 默认使用weaviate + } + + VectorStoreStrategy strategy = strategyMap.get(vectorStoreType); + if (strategy == null) { + log.warn("未找到向量库策略: {}, 使用默认策略: weaviate", vectorStoreType); + strategy = strategyMap.get("weaviate"); + } + + if (strategy == null) { + throw new RuntimeException("未找到可用的向量库策略实现"); + } + + return strategy; + } + + /** + * 根据类型获取特定的向量库策略 + */ + public VectorStoreStrategy getStrategy(String vectorStoreType) { + VectorStoreStrategy strategy = strategyMap.get(vectorStoreType); + if (strategy == null) { + throw new RuntimeException("未找到向量库策略: " + vectorStoreType); + } + return strategy; + } + + /** + * 获取所有可用的向量库类型 + */ + public String[] getAvailableTypes() { + return strategyMap.keySet().toArray(new String[0]); + } +} \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java new file mode 100644 index 00000000..25605629 --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java @@ -0,0 +1,312 @@ +package org.ruoyi.service.strategy.impl; + +import com.google.protobuf.ServiceException; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.model.embedding.EmbeddingModel; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.common.core.service.ConfigService; +import org.ruoyi.domain.bo.QueryVectorBo; +import org.ruoyi.domain.bo.StoreEmbeddingBo; +import org.ruoyi.service.strategy.AbstractVectorStoreStrategy; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Milvus向量库策略实现 + * + * @author ageer + */ +@Slf4j +@Component +public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy { + + // Milvus客户端和相关配置 + // private MilvusClient milvusClient; + + public MilvusVectorStoreStrategy(ConfigService configService) { + super(configService); + } + + @Override + public String getVectorStoreType() { + return "milvus"; + } + + @Override + public void createSchema(String vectorModelName, String kid, String modelName) { + log.info("Milvus创建schema: vectorModelName={}, kid={}, modelName={}", vectorModelName, kid, modelName); + + // 1. 获取Milvus配置 + String host = configService.getConfigValue("milvus", "host"); + String port = configService.getConfigValue("milvus", "port"); + String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; + + // 2. 初始化Milvus客户端 + // ConnectParam connectParam = ConnectParam.newBuilder() + // .withHost(host) + // .withPort(Integer.parseInt(port)) + // .build(); + // milvusClient = new MilvusClient(connectParam); + + // 3. 检查集合是否存在，如果不存在则创建 + // HasCollectionParam hasCollectionParam = HasCollectionParam.newBuilder() + // .withCollectionName(collectionName) + // .build(); + // R hasCollectionResponse = milvusClient.hasCollection(hasCollectionParam); + // + // if (!hasCollectionResponse.getData()) { + // // 创建集合 + // List fieldsSchema = new ArrayList<>(); + // + // // 主键字段 + // fieldsSchema.add(FieldType.newBuilder() + // .withName("id") + // .withDataType(DataType.Int64) + // .withPrimaryKey(true) + // .withAutoID(true) + // .build()); + // + // // 文本字段 + // fieldsSchema.add(FieldType.newBuilder() + // .withName("text") + // .withDataType(DataType.VarChar) + // .withMaxLength(65535) + // .build()); + // + // // fid字段 + // fieldsSchema.add(FieldType.newBuilder() + // .withName("fid") + // .withDataType(DataType.VarChar) + // .withMaxLength(255) + // .build()); + // + // // kid字段 + // fieldsSchema.add(FieldType.newBuilder() + // .withName("kid") + // .withDataType(DataType.VarChar) + // .withMaxLength(255) + // .build()); + // + // // docId字段 + // fieldsSchema.add(FieldType.newBuilder() + // .withName("docId") + // .withDataType(DataType.VarChar) + // .withMaxLength(255) + // .build()); + // + // // 向量字段 + // fieldsSchema.add(FieldType.newBuilder() + // .withName("vector") + // .withDataType(DataType.FloatVector) + // .withDimension(1536) // 根据实际embedding维度调整 + // .build()); + // + // CreateCollectionParam createCollectionParam = CreateCollectionParam.newBuilder() + // .withCollectionName(collectionName) + // .withDescription("Knowledge base collection for " + kid) + // .withShardsNum(2) + // .withFieldTypes(fieldsSchema) + // .build(); + // + // R createCollectionResponse = milvusClient.createCollection(createCollectionParam); + // if (createCollectionResponse.getStatus() == R.Status.Success.getCode()) { + // log.info("Milvus集合创建成功: {}", collectionName); + // + // // 创建索引 + // IndexParam indexParam = IndexParam.newBuilder() + // .withCollectionName(collectionName) + // .withFieldName("vector") + // .withIndexType(IndexType.IVF_FLAT) + // .withMetricType(MetricType.L2) + // .withExtraParam("{\"nlist\":1024}") + // .build(); + // + // R createIndexResponse = milvusClient.createIndex(indexParam); + // if (createIndexResponse.getStatus() == R.Status.Success.getCode()) { + // log.info("Milvus索引创建成功: {}", collectionName); + // } else { + // log.error("Milvus索引创建失败: {}", createIndexResponse.getMessage()); + // } + // } else { + // log.error("Milvus集合创建失败: {}", createCollectionResponse.getMessage()); + // } + // } + + log.info("Milvus schema创建完成: {}", collectionName); + } + + @Override + public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) { + createSchema(storeEmbeddingBo.getVectorModelName(), storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName()); + + EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), + storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl()); + + List chunkList = storeEmbeddingBo.getChunkList(); + List fidList = storeEmbeddingBo.getFids(); + String kid = storeEmbeddingBo.getKid(); + String docId = storeEmbeddingBo.getDocId(); + String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; + + log.info("Milvus向量存储条数记录: " + chunkList.size()); + long startTime = System.currentTimeMillis(); + + // List fields = new ArrayList<>(); + // List textList = new ArrayList<>(); + // List fidListData = new ArrayList<>(); + // List kidList = new ArrayList<>(); + // List docIdList = new ArrayList<>(); + // List> vectorList = new ArrayList<>(); + + for (int i = 0; i < chunkList.size(); i++) { + String text = chunkList.get(i); + String fid = fidList.get(i); + Embedding embedding = embeddingModel.embed(text).content(); + + // textList.add(text); + // fidListData.add(fid); + // kidList.add(kid); + // docIdList.add(docId); + // + // List vector = new ArrayList<>(); + // for (float f : embedding.vector()) { + // vector.add(f); + // } + // vectorList.add(vector); + } + + // fields.add(new InsertParam.Field("text", textList)); + // fields.add(new InsertParam.Field("fid", fidListData)); + // fields.add(new InsertParam.Field("kid", kidList)); + // fields.add(new InsertParam.Field("docId", docIdList)); + // fields.add(new InsertParam.Field("vector", vectorList)); + // + // InsertParam insertParam = InsertParam.newBuilder() + // .withCollectionName(collectionName) + // .withFields(fields) + // .build(); + // + // R insertResponse = milvusClient.insert(insertParam); + // if (insertResponse.getStatus() == R.Status.Success.getCode()) { + // log.info("Milvus向量存储成功，插入条数: {}", insertResponse.getData().getInsertCnt()); + // } else { + // log.error("Milvus向量存储失败: {}", insertResponse.getMessage()); + // throw new ServiceException("Milvus向量存储失败: " + insertResponse.getMessage()); + // } + + long endTime = System.currentTimeMillis(); + log.info("Milvus向量存储完成消耗时间：" + (endTime - startTime) / 1000 + "秒"); + } + + @Override + public List getQueryVector(QueryVectorBo queryVectorBo) { + createSchema(queryVectorBo.getVectorModelName(), queryVectorBo.getKid(), queryVectorBo.getVectorModelName()); + + EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(), + queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl()); + + Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content(); + String collectionName = configService.getConfigValue("milvus", "collectionname") + queryVectorBo.getKid(); + + List resultList = new ArrayList<>(); + + // List searchOutputFields = List.of("text", "fid", "kid", "docId"); + // List> searchVectors = new ArrayList<>(); + // List queryVector = new ArrayList<>(); + // for (float f : queryEmbedding.vector()) { + // queryVector.add(f); + // } + // searchVectors.add(queryVector); + // + // SearchParam searchParam = SearchParam.newBuilder() + // .withCollectionName(collectionName) + // .withMetricType(MetricType.L2) + // .withOutFields(searchOutputFields) + // .withTopK(queryVectorBo.getMaxResults()) + // .withVectors(searchVectors) + // .withVectorFieldName("vector") + // .withParams("{\"nprobe\":10}") + // .build(); + // + // R searchResponse = milvusClient.search(searchParam); + // if (searchResponse.getStatus() == R.Status.Success.getCode()) { + // SearchResults searchResults = searchResponse.getData(); + // List queryResults = searchResults.getResults(); + // + // for (SearchResults.QueryResult queryResult : queryResults) { + // List rows = queryResult.getRows(); + // for (SearchResults.QueryResult.Row row : rows) { + // String text = (String) row.get("text"); + // resultList.add(text); + // } + // } + // } else { + // log.error("Milvus查询失败: {}", searchResponse.getMessage()); + // } + + return resultList; + } + + @Override + public void removeById(String id, String modelName) { + String collectionName = configService.getConfigValue("milvus", "collectionname") + id; + + // DropCollectionParam dropCollectionParam = DropCollectionParam.newBuilder() + // .withCollectionName(collectionName) + // .build(); + // + // R dropResponse = milvusClient.dropCollection(dropCollectionParam); + // if (dropResponse.getStatus() == R.Status.Success.getCode()) { + // log.info("Milvus集合删除成功: {}", collectionName); + // } else { + // log.error("Milvus集合删除失败: {}", dropResponse.getMessage()); + // throw new ServiceException("Milvus集合删除失败: " + dropResponse.getMessage()); + // } + + log.info("Milvus删除集合: {}", collectionName); + } + + @Override + public void removeByDocId(String docId, String kid) { + String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; + + // String expr = "docId == \"" + docId + "\""; + // DeleteParam deleteParam = DeleteParam.newBuilder() + // .withCollectionName(collectionName) + // .withExpr(expr) + // .build(); + // + // R deleteResponse = milvusClient.delete(deleteParam); + // if (deleteResponse.getStatus() == R.Status.Success.getCode()) { + // log.info("Milvus成功删除 docId={} 的所有向量数据，删除条数: {}", docId, deleteResponse.getData().getDeleteCnt()); + // } else { + // log.error("Milvus删除失败: {}", deleteResponse.getMessage()); + // } + + log.info("Milvus删除docId={}的数据", docId); + } + + @Override + public void removeByFid(String fid, String kid) { + String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; + + // String expr = "fid == \"" + fid + "\""; + // DeleteParam deleteParam = DeleteParam.newBuilder() + // .withCollectionName(collectionName) + // .withExpr(expr) + // .build(); + // + // R deleteResponse = milvusClient.delete(deleteParam); + // if (deleteResponse.getStatus() == R.Status.Success.getCode()) { + // log.info("Milvus成功删除 fid={} 的所有向量数据，删除条数: {}", fid, deleteResponse.getData().getDeleteCnt()); + // } else { + // log.error("Milvus删除失败: {}", deleteResponse.getMessage()); + // } + + log.info("Milvus删除fid={}的数据", fid); + } +} \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/WeaviateVectorStoreStrategy.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/WeaviateVectorStoreStrategy.java new file mode 100644 index 00000000..9e3d3aec --- /dev/null +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/WeaviateVectorStoreStrategy.java @@ -0,0 +1,233 @@ +package org.ruoyi.service.strategy.impl; + +import cn.hutool.json.JSONObject; +import com.google.protobuf.ServiceException; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.model.embedding.EmbeddingModel; +import io.weaviate.client.Config; +import io.weaviate.client.WeaviateClient; +import io.weaviate.client.base.Result; +import io.weaviate.client.v1.batch.api.ObjectsBatchDeleter; +import io.weaviate.client.v1.batch.model.BatchDeleteResponse; +import io.weaviate.client.v1.filters.Operator; +import io.weaviate.client.v1.filters.WhereFilter; +import io.weaviate.client.v1.graphql.model.GraphQLResponse; +import io.weaviate.client.v1.schema.model.Property; +import io.weaviate.client.v1.schema.model.Schema; +import io.weaviate.client.v1.schema.model.WeaviateClass; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.common.core.service.ConfigService; +import org.ruoyi.domain.bo.QueryVectorBo; +import org.ruoyi.domain.bo.StoreEmbeddingBo; +import org.ruoyi.service.strategy.AbstractVectorStoreStrategy; +import org.springframework.stereotype.Component; +import java.util.*; + +/** + * Weaviate向量库策略实现 + * + * @author ageer + */ +@Slf4j +@Component +public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy { + + private WeaviateClient client; + + public WeaviateVectorStoreStrategy(ConfigService configService) { + super(configService); + } + + @Override + public String getVectorStoreType() { + return "weaviate"; + } + + @Override + public void createSchema(String vectorModelName, String kid, String modelName) { + String protocol = configService.getConfigValue("weaviate", "protocol"); + String host = configService.getConfigValue("weaviate", "host"); + String className = configService.getConfigValue("weaviate", "classname") + kid; + // 创建 Weaviate 客户端 + client = new WeaviateClient(new Config(protocol, host)); + // 检查类是否存在，如果不存在就创建 schema + Result schemaResult = client.schema().getter().run(); + Schema schema = schemaResult.getResult(); + boolean classExists = false; + for (WeaviateClass weaviateClass : schema.getClasses()) { + if (weaviateClass.getClassName().equals(className)) { + classExists = true; + break; + } + } + if (!classExists) { + // 类不存在，创建 schema + WeaviateClass build = WeaviateClass.builder() + .className(className) + .vectorizer("none") + .properties( + List.of(Property.builder().name("text").dataType(Collections.singletonList("text")).build(), + Property.builder().name("fid").dataType(Collections.singletonList("text")).build(), + Property.builder().name("kid").dataType(Collections.singletonList("text")).build(), + Property.builder().name("docId").dataType(Collections.singletonList("text")).build()) + ) + .build(); + Result createResult = client.schema().classCreator().withClass(build).run(); + if (createResult.hasErrors()) { + log.error("Schema 创建失败: {}", createResult.getError()); + } else { + log.info("Schema 创建成功: {}", className); + } + } + } + + @Override + public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) { + createSchema(storeEmbeddingBo.getVectorModelName(), storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName()); + EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), + storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl()); + List chunkList = storeEmbeddingBo.getChunkList(); + List fidList = storeEmbeddingBo.getFids(); + String kid = storeEmbeddingBo.getKid(); + String docId = storeEmbeddingBo.getDocId(); + log.info("向量存储条数记录: " + chunkList.size()); + long startTime = System.currentTimeMillis(); + for (int i = 0; i < chunkList.size(); i++) { + String text = chunkList.get(i); + String fid = fidList.get(i); + Embedding embedding = embeddingModel.embed(text).content(); + Map properties = Map.of( + "text", text, + "fid", fid, + "kid", kid, + "docId", docId + ); + Float[] vector = toObjectArray(embedding.vector()); + client.data().creator() + .withClassName("LocalKnowledge" + kid) + .withProperties(properties) + .withVector(vector) + .run(); + } + long endTime = System.currentTimeMillis(); + log.info("向量存储完成消耗时间：" + (endTime - startTime) / 1000 + "秒"); + } + + + + @Override + public List getQueryVector(QueryVectorBo queryVectorBo) { + createSchema(queryVectorBo.getVectorModelName(), queryVectorBo.getKid(), queryVectorBo.getVectorModelName()); + EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(), + queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl()); + Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content(); + float[] vector = queryEmbedding.vector(); + List vectorStrings = new ArrayList<>(); + for (float v : vector) { + vectorStrings.add(String.valueOf(v)); + } + String vectorStr = String.join(",", vectorStrings); + String className = configService.getConfigValue("weaviate", "classname"); + + // 构建 GraphQL 查询 + String graphQLQuery = String.format( + "{\n" + + " Get {\n" + + " %s(nearVector: {vector: [%s]} limit: %d) {\n" + + " text\n" + + " fid\n" + + " kid\n" + + " docId\n" + + " _additional {\n" + + " distance\n" + + " id\n" + + " }\n" + + " }\n" + + " }\n" + + "}", + className + queryVectorBo.getKid(), + vectorStr, + queryVectorBo.getMaxResults() + ); + + Result result = client.graphQL().raw().withQuery(graphQLQuery).run(); + List resultList = new ArrayList<>(); + if (result != null && !result.hasErrors()) { + Object data = result.getResult().getData(); + JSONObject entries = new JSONObject(data); + Map entriesMap = entries.get("Get", Map.class); + cn.hutool.json.JSONArray objects = entriesMap.get(className + queryVectorBo.getKid()); + if (objects.isEmpty()) { + return resultList; + } + for (Object object : objects) { + Map map = (Map) object; + String content = map.get("text"); + resultList.add(content); + } + return resultList; + } else { + log.error("GraphQL 查询失败: {}", result.getError()); + return resultList; + } + } + + @Override + @SneakyThrows + public void removeById(String id, String modelName) { + String protocol = configService.getConfigValue("weaviate", "protocol"); + String host = configService.getConfigValue("weaviate", "host"); + String className = configService.getConfigValue("weaviate", "classname"); + String finalClassName = className + id; + WeaviateClient client = new WeaviateClient(new Config(protocol, host)); + Result result = client.schema().classDeleter().withClassName(finalClassName).run(); + if (result.hasErrors()) { + log.error("失败删除向量: " + result.getError()); + throw new ServiceException("失败删除向量数据!"); + } else { + log.info("成功删除向量数据: " + result.getResult()); + } + } + + @Override + public void removeByDocId(String docId, String kid) { + String className = configService.getConfigValue("weaviate", "classname") + kid; + // 构建 Where 条件 + WhereFilter whereFilter = WhereFilter.builder() + .path("docId") + .operator(Operator.Equal) + .valueText(docId) + .build(); + ObjectsBatchDeleter deleter = client.batch().objectsBatchDeleter(); + Result result = deleter.withClassName(className) + .withWhere(whereFilter) + .run(); + if (result != null && !result.hasErrors()) { + log.info("成功删除 docId={} 的所有向量数据", docId); + } else { + log.error("删除失败: {}", result.getError()); + } + } + + @Override + public void removeByFid(String fid, String kid) { + String className = configService.getConfigValue("weaviate", "classname") + kid; + // 构建 Where 条件 + WhereFilter whereFilter = WhereFilter.builder() + .path("fid") + .operator(Operator.Equal) + .valueText(fid) + .build(); + ObjectsBatchDeleter deleter = client.batch().objectsBatchDeleter(); + Result result = deleter.withClassName(className) + .withWhere(whereFilter) + .run(); + if (result != null && !result.hasErrors()) { + log.info("成功删除 fid={} 的所有向量数据", fid); + } else { + log.error("删除失败: {}", result.getError()); + } + } + +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/KnowledgeInfoServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/KnowledgeInfoServiceImpl.java index a5be768b..23148c44 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/KnowledgeInfoServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/KnowledgeInfoServiceImpl.java @@ -216,7 +216,7 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { } baseMapper.insert(knowledgeInfo); if (knowledgeInfo != null) { - vectorStoreService.createSchema(String.valueOf(knowledgeInfo.getId()), + vectorStoreService.createSchema(knowledgeInfo.getVectorModelName(),String.valueOf(knowledgeInfo.getId()), bo.getVectorModelName()); } } else { @@ -257,6 +257,7 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { knowledgeAttach.setDocType(fileName.substring(fileName.lastIndexOf(".") + 1)); String content = ""; ResourceLoader resourceLoader = resourceLoaderFactory.getLoaderByFileType(knowledgeAttach.getDocType()); + // 文档分段入库 List fids = new ArrayList<>(); try { content = resourceLoader.getContent(file.getInputStream()); @@ -264,6 +265,7 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { List knowledgeFragmentList = new ArrayList<>(); if (CollUtil.isNotEmpty(chunkList)) { for (int i = 0; i < chunkList.size(); i++) { + // 生成知识片段ID String fid = RandomUtil.randomString(10); fids.add(fid); KnowledgeFragment knowledgeFragment = new KnowledgeFragment(); From ef49429543b8f593d3ebdcbdc8569a065665962e Mon Sep 17 00:00:00 2001 From: Yzm Date: Mon, 29 Sep 2025 18:36:48 +0800 Subject: [PATCH 2/6] =?UTF-8?q?feat(milvus):=20=E5=AE=9E=E7=8E=B0Milvus?= =?UTF-8?q?=E5=90=91=E9=87=8F=E6=95=B0=E6=8D=AE=E5=BA=93=E9=9B=86=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 添加Milvus Java SDK依赖 - 实现MilvusVectorStoreStrategy核心功能 - 支持集合管理、数据存储、向量搜索和数据删除 - 添加Milvus实现指南文档 - 更新数据库连接配置 - 修改VectorStoreService接口添加异常声明 --- MILVUS_IMPLEMENTATION_GUIDE.md | 237 ++++++++++ .../src/main/resources/application-dev.yml | 4 +- ruoyi-modules-api/ruoyi-knowledge-api/pom.xml | 6 + .../org/ruoyi/service/VectorStoreService.java | 9 +- .../impl/MilvusVectorStoreStrategy.java | 417 +++++++++--------- 5 files changed, 460 insertions(+), 213 deletions(-) create mode 100644 MILVUS_IMPLEMENTATION_GUIDE.md diff --git a/MILVUS_IMPLEMENTATION_GUIDE.md b/MILVUS_IMPLEMENTATION_GUIDE.md new file mode 100644 index 00000000..181a52ec --- /dev/null +++ b/MILVUS_IMPLEMENTATION_GUIDE.md @@ -0,0 +1,237 @@ +# Milvus向量库实现指南 + +## 概述 + +本项目已完成Milvus向量库的集成，基于Milvus SDK 2.6.4版本实现。Milvus是一个开源的向量数据库，专为AI应用和相似性搜索而设计。 + +## 实现特性 + +### ✅ 已实现功能 + +1. **集合管理** + - 自动创建集合（Collection） + - 检查集合是否存在 + - 删除集合 + +2. **数据存储** + - 批量插入向量数据 + - 支持文本、fid、kid、docId等元数据 + - 自动生成向量嵌入 + +3. **向量搜索** + - 基于相似性的向量搜索 + - 支持TopK结果返回 + - 返回相关文本内容 + +4. **数据删除** + - 按文档ID删除 + - 按片段ID删除 + - 删除整个集合 + +## 架构设计 + +### 策略模式实现 + +``` +AbstractVectorStoreStrategy (抽象基类) + ↓ +MilvusVectorStoreStrategy (Milvus实现) +WeaviateVectorStoreStrategy (Weaviate实现) +``` + +### 核心类说明 + +- **MilvusVectorStoreStrategy**: Milvus向量库策略实现 +- **VectorStoreStrategyFactory**: 向量库策略工厂，支持动态切换 +- **VectorStoreService**: 向量库服务接口 + +## 配置说明 + +### 必需配置项 + +在系统配置中需要设置以下Milvus相关配置： + +```properties +# Milvus服务地址 +milvus.url=http://localhost:19530 + +# 集合名称前缀 +milvus.collectionname=LocalKnowledge + +# 向量库类型选择 +vector.store_type=milvus +``` + +### 集合Schema设计 + +每个集合包含以下字段： + +| 字段名 | 类型 | 说明 | +|--------|------|------| +| id | Int64 | 主键，自动生成 | +| text | VarChar(65535) | 文本内容 | +| fid | VarChar(255) | 片段ID | +| kid | VarChar(255) | 知识库ID | +| docId | VarChar(255) | 文档ID | +| vector | FloatVector(1024) | 向量数据 | + +### 索引配置 + +- **索引类型**: IVF_FLAT +- **距离度量**: L2 (欧几里得距离) +- **参数**: nlist=1024 + +## 使用示例 + +### 1. 创建集合 + +```java +MilvusVectorStoreStrategy strategy = new MilvusVectorStoreStrategy(configService); +strategy.createSchema("bge-large-zh-v1.5", "test001", "test-model"); +``` + +### 2. 存储向量数据 + +```java +StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo(); +storeEmbeddingBo.setVectorModelName("bge-large-zh-v1.5"); +storeEmbeddingBo.setKid("test001"); +storeEmbeddingBo.setDocId("doc001"); +storeEmbeddingBo.setChunkList(Arrays.asList("文本1", "文本2")); +storeEmbeddingBo.setFids(Arrays.asList("fid001", "fid002")); + +strategy.storeEmbeddings(storeEmbeddingBo); +``` + +### 3. 查询向量数据 + +```java +QueryVectorBo queryVectorBo = new QueryVectorBo(); +queryVectorBo.setQuery("查询文本"); +queryVectorBo.setKid("test001"); +queryVectorBo.setMaxResults(5); + +List results = strategy.getQueryVector(queryVectorBo); +``` + +### 4. 删除数据 + +```java +// 按文档ID删除 +strategy.removeByDocId("doc001", "test001"); + +// 按片段ID删除 +strategy.removeByFid("fid001", "test001"); + +// 删除整个集合 +strategy.removeById("test001", "model"); +``` + +## 部署要求 + +### Milvus服务部署 + +1. **Docker部署** (推荐) +```bash +# 下载docker-compose文件 +wget https://github.com/milvus-io/milvus/releases/download/v2.6.4/milvus-standalone-docker-compose.yml -O docker-compose.yml + +# 启动Milvus +docker-compose up -d +``` + +2. **验证部署** +```bash +# 检查服务状态 +docker-compose ps + +# 查看日志 +docker-compose logs milvus-standalone +``` + +### 系统要求 + +- **内存**: 最少8GB，推荐16GB+ +- **存储**: SSD推荐，至少50GB可用空间 +- **CPU**: 4核心以上 +- **网络**: 确保19530端口可访问 + +## 性能优化 + +### 1. 索引优化 + +根据数据量调整索引参数： +- 小数据集(<100万): nlist=1024 +- 中等数据集(100万-1000万): nlist=4096 +- 大数据集(>1000万): nlist=16384 + +### 2. 批量操作 + +- 批量插入：建议每批1000-10000条记录 +- 批量查询：避免频繁的单条查询 + +### 3. 内存管理 + +```yaml +# docker-compose.yml中的内存配置 +environment: + MILVUS_CONFIG_PATH: /milvus/configs/milvus.yaml +volumes: + - ./milvus.yaml:/milvus/configs/milvus.yaml +``` + +## 故障排除 + +### 常见问题 + +1. **连接失败** + - 检查Milvus服务是否启动 + - 验证网络连接和端口 + - 确认配置中的URL正确 + +2. **集合创建失败** + - 检查集合名称是否符合规范 + - 验证字段定义是否正确 + - 查看Milvus日志获取详细错误 + +3. **插入数据失败** + - 确认向量维度与schema一致 + - 检查数据格式是否正确 + - 验证集合是否已加载 + +4. **查询无结果** + - 确认集合中有数据 + - 检查查询参数设置 + - 验证向量化模型一致性 + +### 日志调试 + +启用详细日志： +```properties +logging.level.org.ruoyi.service.strategy.impl.MilvusVectorStoreStrategy=DEBUG +logging.level.io.milvus=DEBUG +``` + +## 与Weaviate对比 + +| 特性 | Milvus | Weaviate | +|------|--------|----------| +| 性能 | 高性能，专为大规模设计 | 中等性能 | +| 部署 | 需要独立部署 | 可独立部署或云服务 | +| 生态 | 专注向量搜索 | 集成更多AI功能 | +| 学习成本 | 中等 | 较低 | +| 扩展性 | 优秀 | 良好 | + +## 后续优化建议 + +1. **连接池管理**: 实现MilvusClient连接池 +2. **异步操作**: 支持异步插入和查询 +3. **分片策略**: 大数据集的分片管理 +4. **监控告警**: 集成性能监控 +5. **备份恢复**: 数据备份和恢复机制 + +## 参考资料 + +- [Milvus官方文档](https://milvus.io/docs) +- [Milvus Java SDK](https://github.com/milvus-io/milvus-sdk-java) +- [向量数据库最佳实践](https://milvus.io/docs/performance_faq.md) \ No newline at end of file diff --git a/ruoyi-admin/src/main/resources/application-dev.yml b/ruoyi-admin/src/main/resources/application-dev.yml index 1fd51a25..c6e4e3f0 100644 --- a/ruoyi-admin/src/main/resources/application-dev.yml +++ b/ruoyi-admin/src/main/resources/application-dev.yml @@ -16,9 +16,9 @@ spring: master: type: ${spring.datasource.type} driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/ruoyi-ai?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true + url: jdbc:mysql://127.0.0.1:3306/ruoyi-ai-github?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true username: root - password: root + password: qxyg1010 hikari: # 最大连接池数量 diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml b/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml index f8082a67..40a075e3 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml +++ b/ruoyi-modules-api/ruoyi-knowledge-api/pom.xml @@ -74,6 +74,12 @@ 1.19.6 + + io.milvus + milvus-sdk-java + 2.6.4 + + dev.langchain4j langchain4j-open-ai diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java index 21d2410d..ae0a227d 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java @@ -1,5 +1,6 @@ package org.ruoyi.service; +import com.google.protobuf.ServiceException; import org.ruoyi.domain.bo.QueryVectorBo; import org.ruoyi.domain.bo.StoreEmbeddingBo; @@ -11,15 +12,15 @@ import java.util.List; */ public interface VectorStoreService { - void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo); + void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) throws ServiceException; List getQueryVector(QueryVectorBo queryVectorBo); void createSchema(String vectorModelName, String kid,String modelName); - void removeById(String id,String modelName); + void removeById(String id,String modelName) throws ServiceException; - void removeByDocId(String docId, String kid); + void removeByDocId(String docId, String kid) throws ServiceException; - void removeByFid(String fid, String kid); + void removeByFid(String fid, String kid) throws ServiceException; } diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java index 25605629..540eefc8 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java @@ -1,8 +1,28 @@ package org.ruoyi.service.strategy.impl; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import com.google.protobuf.ServiceException; import dev.langchain4j.data.embedding.Embedding; import dev.langchain4j.model.embedding.EmbeddingModel; +import io.milvus.v2.client.ConnectConfig; +import io.milvus.v2.client.MilvusClientV2; +import io.milvus.v2.common.DataType; +import io.milvus.v2.common.IndexParam; +import io.milvus.v2.service.collection.request.AddFieldReq; +import io.milvus.v2.service.collection.request.CreateCollectionReq; +import io.milvus.v2.service.collection.request.DescribeCollectionReq; +import io.milvus.v2.service.collection.request.DropCollectionReq; +import io.milvus.v2.service.collection.request.HasCollectionReq; +import io.milvus.v2.service.collection.response.DescribeCollectionResp; +import io.milvus.v2.service.vector.request.DeleteReq; +import io.milvus.v2.service.vector.request.InsertReq; +import io.milvus.v2.service.vector.request.SearchReq; +import io.milvus.v2.service.vector.request.data.BaseVector; +import io.milvus.v2.service.vector.request.data.FloatVec; +import io.milvus.v2.service.vector.response.DeleteResp; +import io.milvus.v2.service.vector.response.InsertResp; +import io.milvus.v2.service.vector.response.SearchResp; import lombok.extern.slf4j.Slf4j; import org.ruoyi.common.core.service.ConfigService; import org.ruoyi.domain.bo.QueryVectorBo; @@ -10,10 +30,7 @@ import org.ruoyi.domain.bo.StoreEmbeddingBo; import org.ruoyi.service.strategy.AbstractVectorStoreStrategy; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Milvus向量库策略实现 @@ -24,8 +41,7 @@ import java.util.Map; @Component public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy { - // Milvus客户端和相关配置 - // private MilvusClient milvusClient; + private MilvusClientV2 client; public MilvusVectorStoreStrategy(ConfigService configService) { super(configService); @@ -41,106 +57,89 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy { log.info("Milvus创建schema: vectorModelName={}, kid={}, modelName={}", vectorModelName, kid, modelName); // 1. 获取Milvus配置 - String host = configService.getConfigValue("milvus", "host"); - String port = configService.getConfigValue("milvus", "port"); + String host = configService.getConfigValue("milvus", "url"); String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; - // 2. 初始化Milvus客户端 - // ConnectParam connectParam = ConnectParam.newBuilder() - // .withHost(host) - // .withPort(Integer.parseInt(port)) - // .build(); - // milvusClient = new MilvusClient(connectParam); + ConnectConfig config = ConnectConfig.builder() + .uri(host) + .build(); + client = new MilvusClientV2(config); + + // 2. 检查集合是否存在 + HasCollectionReq hasCollectionReq = HasCollectionReq.builder() + .collectionName(collectionName) + .build(); - // 3. 检查集合是否存在，如果不存在则创建 - // HasCollectionParam hasCollectionParam = HasCollectionParam.newBuilder() - // .withCollectionName(collectionName) - // .build(); - // R hasCollectionResponse = milvusClient.hasCollection(hasCollectionParam); - // - // if (!hasCollectionResponse.getData()) { - // // 创建集合 - // List fieldsSchema = new ArrayList<>(); - // - // // 主键字段 - // fieldsSchema.add(FieldType.newBuilder() - // .withName("id") - // .withDataType(DataType.Int64) - // .withPrimaryKey(true) - // .withAutoID(true) - // .build()); - // - // // 文本字段 - // fieldsSchema.add(FieldType.newBuilder() - // .withName("text") - // .withDataType(DataType.VarChar) - // .withMaxLength(65535) - // .build()); - // - // // fid字段 - // fieldsSchema.add(FieldType.newBuilder() - // .withName("fid") - // .withDataType(DataType.VarChar) - // .withMaxLength(255) - // .build()); - // - // // kid字段 - // fieldsSchema.add(FieldType.newBuilder() - // .withName("kid") - // .withDataType(DataType.VarChar) - // .withMaxLength(255) - // .build()); - // - // // docId字段 - // fieldsSchema.add(FieldType.newBuilder() - // .withName("docId") - // .withDataType(DataType.VarChar) - // .withMaxLength(255) - // .build()); - // - // // 向量字段 - // fieldsSchema.add(FieldType.newBuilder() - // .withName("vector") - // .withDataType(DataType.FloatVector) - // .withDimension(1536) // 根据实际embedding维度调整 - // .build()); - // - // CreateCollectionParam createCollectionParam = CreateCollectionParam.newBuilder() - // .withCollectionName(collectionName) - // .withDescription("Knowledge base collection for " + kid) - // .withShardsNum(2) - // .withFieldTypes(fieldsSchema) - // .build(); - // - // R createCollectionResponse = milvusClient.createCollection(createCollectionParam); - // if (createCollectionResponse.getStatus() == R.Status.Success.getCode()) { - // log.info("Milvus集合创建成功: {}", collectionName); - // - // // 创建索引 - // IndexParam indexParam = IndexParam.newBuilder() - // .withCollectionName(collectionName) - // .withFieldName("vector") - // .withIndexType(IndexType.IVF_FLAT) - // .withMetricType(MetricType.L2) - // .withExtraParam("{\"nlist\":1024}") - // .build(); - // - // R createIndexResponse = milvusClient.createIndex(indexParam); - // if (createIndexResponse.getStatus() == R.Status.Success.getCode()) { - // log.info("Milvus索引创建成功: {}", collectionName); - // } else { - // log.error("Milvus索引创建失败: {}", createIndexResponse.getMessage()); - // } - // } else { - // log.error("Milvus集合创建失败: {}", createCollectionResponse.getMessage()); - // } - // } + Boolean hasCollection = client.hasCollection(hasCollectionReq); - log.info("Milvus schema创建完成: {}", collectionName); + if (!hasCollection) { + // 3. 创建集合schema + CreateCollectionReq.CollectionSchema schema = CreateCollectionReq.CollectionSchema.builder() + .build(); + + // 添加字段定义 + schema.addField(AddFieldReq.builder() + .fieldName("id") + .dataType(DataType.Int64) + .isPrimaryKey(true) + .autoID(true) + .build()); + + schema.addField(AddFieldReq.builder() + .fieldName("text") + .dataType(DataType.VarChar) + .maxLength(65535) + .build()); + + schema.addField(AddFieldReq.builder() + .fieldName("fid") + .dataType(DataType.VarChar) + .maxLength(255) + .build()); + + schema.addField(AddFieldReq.builder() + .fieldName("kid") + .dataType(DataType.VarChar) + .maxLength(255) + .build()); + + schema.addField(AddFieldReq.builder() + .fieldName("docId") + .dataType(DataType.VarChar) + .maxLength(255) + .build()); + + schema.addField(AddFieldReq.builder() + .fieldName("vector") + .dataType(DataType.FloatVector) + .dimension(1024) // 根据实际embedding维度调整 + .build()); + + // 4. 创建索引参数 + List indexParams = new ArrayList<>(); + indexParams.add(IndexParam.builder() + .fieldName("vector") + .indexType(IndexParam.IndexType.IVF_FLAT) + .metricType(IndexParam.MetricType.L2) + .extraParams(Map.of("nlist", 1024)) + .build()); + + // 5. 创建集合 + CreateCollectionReq createCollectionReq = CreateCollectionReq.builder() + .collectionName(collectionName) + .collectionSchema(schema) + .indexParams(indexParams) + .build(); + + client.createCollection(createCollectionReq); + log.info("Milvus集合创建成功: {}", collectionName); + } else { + log.info("Milvus集合已存在: {}", collectionName); + } } @Override - public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) { + public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) throws ServiceException { createSchema(storeEmbeddingBo.getVectorModelName(), storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName()); EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), @@ -155,48 +154,56 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy { log.info("Milvus向量存储条数记录: " + chunkList.size()); long startTime = System.currentTimeMillis(); - // List fields = new ArrayList<>(); - // List textList = new ArrayList<>(); - // List fidListData = new ArrayList<>(); - // List kidList = new ArrayList<>(); - // List docIdList = new ArrayList<>(); - // List> vectorList = new ArrayList<>(); + // 准备批量插入数据 + List textList = new ArrayList<>(); + List fidListData = new ArrayList<>(); + List kidList = new ArrayList<>(); + List docIdList = new ArrayList<>(); + List> vectorList = new ArrayList<>(); for (int i = 0; i < chunkList.size(); i++) { String text = chunkList.get(i); String fid = fidList.get(i); Embedding embedding = embeddingModel.embed(text).content(); - // textList.add(text); - // fidListData.add(fid); - // kidList.add(kid); - // docIdList.add(docId); - // - // List vector = new ArrayList<>(); - // for (float f : embedding.vector()) { - // vector.add(f); - // } - // vectorList.add(vector); + textList.add(text); + fidListData.add(fid); + kidList.add(kid); + docIdList.add(docId); + + List vector = new ArrayList<>(); + for (float f : embedding.vector()) { + vector.add(f); + } + vectorList.add(vector); } - // fields.add(new InsertParam.Field("text", textList)); - // fields.add(new InsertParam.Field("fid", fidListData)); - // fields.add(new InsertParam.Field("kid", kidList)); - // fields.add(new InsertParam.Field("docId", docIdList)); - // fields.add(new InsertParam.Field("vector", vectorList)); - // - // InsertParam insertParam = InsertParam.newBuilder() - // .withCollectionName(collectionName) - // .withFields(fields) - // .build(); - // - // R insertResponse = milvusClient.insert(insertParam); - // if (insertResponse.getStatus() == R.Status.Success.getCode()) { - // log.info("Milvus向量存储成功，插入条数: {}", insertResponse.getData().getInsertCnt()); - // } else { - // log.error("Milvus向量存储失败: {}", insertResponse.getMessage()); - // throw new ServiceException("Milvus向量存储失败: " + insertResponse.getMessage()); - // } + // 构建插入数据 + List data = new ArrayList<>(); + Gson gson = new Gson(); + for (int i = 0; i < textList.size(); i++) { + JsonObject row = new JsonObject(); + row.addProperty("text", textList.get(i)); + row.addProperty("fid", fidListData.get(i)); + row.addProperty("kid", kidList.get(i)); + row.addProperty("docId", docIdList.get(i)); + row.add("vector", gson.toJsonTree(vectorList.get(i))); + data.add(row); + } + + // 执行插入 + InsertReq insertReq = InsertReq.builder() + .collectionName(collectionName) + .data(data) + .build(); + + InsertResp insertResp = client.insert(insertReq); + if (insertResp.getInsertCnt() > 0) { + log.info("Milvus向量存储成功，插入条数: {}", insertResp.getInsertCnt()); + } else { + log.error("Milvus向量存储失败"); + throw new ServiceException("Milvus向量存储失败"); + } long endTime = System.currentTimeMillis(); log.info("Milvus向量存储完成消耗时间：" + (endTime - startTime) / 1000 + "秒"); @@ -214,99 +221,95 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy { List resultList = new ArrayList<>(); - // List searchOutputFields = List.of("text", "fid", "kid", "docId"); - // List> searchVectors = new ArrayList<>(); - // List queryVector = new ArrayList<>(); - // for (float f : queryEmbedding.vector()) { - // queryVector.add(f); - // } - // searchVectors.add(queryVector); - // - // SearchParam searchParam = SearchParam.newBuilder() - // .withCollectionName(collectionName) - // .withMetricType(MetricType.L2) - // .withOutFields(searchOutputFields) - // .withTopK(queryVectorBo.getMaxResults()) - // .withVectors(searchVectors) - // .withVectorFieldName("vector") - // .withParams("{\"nprobe\":10}") - // .build(); - // - // R searchResponse = milvusClient.search(searchParam); - // if (searchResponse.getStatus() == R.Status.Success.getCode()) { - // SearchResults searchResults = searchResponse.getData(); - // List queryResults = searchResults.getResults(); - // - // for (SearchResults.QueryResult queryResult : queryResults) { - // List rows = queryResult.getRows(); - // for (SearchResults.QueryResult.Row row : rows) { - // String text = (String) row.get("text"); - // resultList.add(text); - // } - // } - // } else { - // log.error("Milvus查询失败: {}", searchResponse.getMessage()); - // } + // 准备查询向量 + List searchVectors = new ArrayList<>(); + float[] queryVectorArray = new float[queryEmbedding.vector().length]; + for (int i = 0; i < queryEmbedding.vector().length; i++) { + queryVectorArray[i] = queryEmbedding.vector()[i]; + } + searchVectors.add(new FloatVec(queryVectorArray)); + + // 构建搜索请求 + SearchReq searchReq = SearchReq.builder() + .collectionName(collectionName) + .data(searchVectors) + .topK(queryVectorBo.getMaxResults()) + .outputFields(Arrays.asList("text", "fid", "kid", "docId")) + .build(); + + SearchResp searchResp = client.search(searchReq); + if (searchResp != null && searchResp.getSearchResults() != null) { + List> searchResults = searchResp.getSearchResults(); + + for (List results : searchResults) { + for (SearchResp.SearchResult result : results) { + Map entity = result.getEntity(); + String text = (String) entity.get("text"); + if (text != null) { + resultList.add(text); + } + } + } + } else { + log.error("Milvus查询失败或无结果"); + } return resultList; } @Override - public void removeById(String id, String modelName) { + public void removeById(String id, String modelName) throws ServiceException { String collectionName = configService.getConfigValue("milvus", "collectionname") + id; - // DropCollectionParam dropCollectionParam = DropCollectionParam.newBuilder() - // .withCollectionName(collectionName) - // .build(); - // - // R dropResponse = milvusClient.dropCollection(dropCollectionParam); - // if (dropResponse.getStatus() == R.Status.Success.getCode()) { - // log.info("Milvus集合删除成功: {}", collectionName); - // } else { - // log.error("Milvus集合删除失败: {}", dropResponse.getMessage()); - // throw new ServiceException("Milvus集合删除失败: " + dropResponse.getMessage()); - // } + // 删除整个集合 + DropCollectionReq dropCollectionReq = DropCollectionReq.builder() + .collectionName(collectionName) + .build(); - log.info("Milvus删除集合: {}", collectionName); + try { + client.dropCollection(dropCollectionReq); + log.info("Milvus集合删除成功: {}", collectionName); + } catch (Exception e) { + log.error("Milvus集合删除失败: {}", e.getMessage()); + throw new ServiceException("Milvus集合删除失败: " + e.getMessage()); + } } @Override - public void removeByDocId(String docId, String kid) { + public void removeByDocId(String docId, String kid) throws ServiceException { String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; - // String expr = "docId == \"" + docId + "\""; - // DeleteParam deleteParam = DeleteParam.newBuilder() - // .withCollectionName(collectionName) - // .withExpr(expr) - // .build(); - // - // R deleteResponse = milvusClient.delete(deleteParam); - // if (deleteResponse.getStatus() == R.Status.Success.getCode()) { - // log.info("Milvus成功删除 docId={} 的所有向量数据，删除条数: {}", docId, deleteResponse.getData().getDeleteCnt()); - // } else { - // log.error("Milvus删除失败: {}", deleteResponse.getMessage()); - // } + String expr = "docId == \"" + docId + "\""; + DeleteReq deleteReq = DeleteReq.builder() + .collectionName(collectionName) + .filter(expr) + .build(); - log.info("Milvus删除docId={}的数据", docId); + try { + DeleteResp deleteResp = client.delete(deleteReq); + log.info("Milvus成功删除 docId={} 的所有向量数据，删除条数: {}", docId, deleteResp.getDeleteCnt()); + } catch (Exception e) { + log.error("Milvus删除失败: {}", e.getMessage()); + throw new ServiceException(e.getMessage()); + } } @Override - public void removeByFid(String fid, String kid) { + public void removeByFid(String fid, String kid) throws ServiceException { String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; - // String expr = "fid == \"" + fid + "\""; - // DeleteParam deleteParam = DeleteParam.newBuilder() - // .withCollectionName(collectionName) - // .withExpr(expr) - // .build(); - // - // R deleteResponse = milvusClient.delete(deleteParam); - // if (deleteResponse.getStatus() == R.Status.Success.getCode()) { - // log.info("Milvus成功删除 fid={} 的所有向量数据，删除条数: {}", fid, deleteResponse.getData().getDeleteCnt()); - // } else { - // log.error("Milvus删除失败: {}", deleteResponse.getMessage()); - // } + String expr = "fid == \"" + fid + "\""; + DeleteReq deleteReq = DeleteReq.builder() + .collectionName(collectionName) + .filter(expr) + .build(); - log.info("Milvus删除fid={}的数据", fid); + try { + DeleteResp deleteResp = client.delete(deleteReq); + log.info("Milvus成功删除 fid={} 的所有向量数据，删除条数: {}", fid, deleteResp.getDeleteCnt()); + } catch (Exception e) { + log.error("Milvus删除失败: {}", e.getMessage()); + throw new ServiceException(e.getMessage()); + } } } \ No newline at end of file From f71cf85dc828104421d3a2a1f646b761f866857c Mon Sep 17 00:00:00 2001 From: Yzm Date: Mon, 29 Sep 2025 21:45:01 +0800 Subject: [PATCH 3/6] =?UTF-8?q?feat(knowledge):=20=E5=AE=9E=E7=8E=B0Milvus?= =?UTF-8?q?=E5=90=91=E9=87=8F=E5=BA=93=E7=AD=96=E7=95=A5=E5=B9=B6=E9=87=8D?= =?UTF-8?q?=E6=9E=84=E9=85=8D=E7=BD=AE=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增Milvus向量库策略实现类MilvusVectorStoreStrategy - 重构向量库配置管理，使用VectorStoreProperties统一配置 - 修改AbstractVectorStoreStrategy抽象类依赖注入方式 - 更新Weaviate策略实现类适配新的配置方式 - 移除旧的ConfigService配置读取方式 - 添加向量库类型配置项，默认使用weaviate - 实现Milvus集合创建、数据存储、向量搜索和删除功能 - 优化向量库策略工厂类VectorStoreStrategyFactory初始化逻辑 - 删除已废弃的Milvus实现指南文档 - 升级Milvus SDK版本并调整相关API调用方式 --- MILVUS_IMPLEMENTATION_GUIDE.md | 237 ----------- .../src/main/resources/application.yml | 14 + .../core/config/VectorStoreProperties.java | 62 +++ .../org/ruoyi/service/VectorStoreService.java | 2 +- .../service/impl/VectorStoreServiceImpl.java | 10 +- .../strategy/AbstractVectorStoreStrategy.java | 6 +- .../strategy/VectorStoreStrategyFactory.java | 78 ++-- .../impl/MilvusVectorStoreStrategy.java | 370 ++++++++++-------- .../impl/WeaviateVectorStoreStrategy.java | 26 +- 9 files changed, 323 insertions(+), 482 deletions(-) delete mode 100644 MILVUS_IMPLEMENTATION_GUIDE.md create mode 100644 ruoyi-common/ruoyi-common-core/src/main/java/org/ruoyi/common/core/config/VectorStoreProperties.java diff --git a/MILVUS_IMPLEMENTATION_GUIDE.md b/MILVUS_IMPLEMENTATION_GUIDE.md deleted file mode 100644 index 181a52ec..00000000 --- a/MILVUS_IMPLEMENTATION_GUIDE.md +++ /dev/null @@ -1,237 +0,0 @@ -# Milvus向量库实现指南 - -## 概述 - -本项目已完成Milvus向量库的集成，基于Milvus SDK 2.6.4版本实现。Milvus是一个开源的向量数据库，专为AI应用和相似性搜索而设计。 - -## 实现特性 - -### ✅ 已实现功能 - -1. **集合管理** - - 自动创建集合（Collection） - - 检查集合是否存在 - - 删除集合 - -2. **数据存储** - - 批量插入向量数据 - - 支持文本、fid、kid、docId等元数据 - - 自动生成向量嵌入 - -3. **向量搜索** - - 基于相似性的向量搜索 - - 支持TopK结果返回 - - 返回相关文本内容 - -4. **数据删除** - - 按文档ID删除 - - 按片段ID删除 - - 删除整个集合 - -## 架构设计 - -### 策略模式实现 - -``` -AbstractVectorStoreStrategy (抽象基类) - ↓ -MilvusVectorStoreStrategy (Milvus实现) -WeaviateVectorStoreStrategy (Weaviate实现) -``` - -### 核心类说明 - -- **MilvusVectorStoreStrategy**: Milvus向量库策略实现 -- **VectorStoreStrategyFactory**: 向量库策略工厂，支持动态切换 -- **VectorStoreService**: 向量库服务接口 - -## 配置说明 - -### 必需配置项 - -在系统配置中需要设置以下Milvus相关配置： - -```properties -# Milvus服务地址 -milvus.url=http://localhost:19530 - -# 集合名称前缀 -milvus.collectionname=LocalKnowledge - -# 向量库类型选择 -vector.store_type=milvus -``` - -### 集合Schema设计 - -每个集合包含以下字段： - -| 字段名 | 类型 | 说明 | -|--------|------|------| -| id | Int64 | 主键，自动生成 | -| text | VarChar(65535) | 文本内容 | -| fid | VarChar(255) | 片段ID | -| kid | VarChar(255) | 知识库ID | -| docId | VarChar(255) | 文档ID | -| vector | FloatVector(1024) | 向量数据 | - -### 索引配置 - -- **索引类型**: IVF_FLAT -- **距离度量**: L2 (欧几里得距离) -- **参数**: nlist=1024 - -## 使用示例 - -### 1. 创建集合 - -```java -MilvusVectorStoreStrategy strategy = new MilvusVectorStoreStrategy(configService); -strategy.createSchema("bge-large-zh-v1.5", "test001", "test-model"); -``` - -### 2. 存储向量数据 - -```java -StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo(); -storeEmbeddingBo.setVectorModelName("bge-large-zh-v1.5"); -storeEmbeddingBo.setKid("test001"); -storeEmbeddingBo.setDocId("doc001"); -storeEmbeddingBo.setChunkList(Arrays.asList("文本1", "文本2")); -storeEmbeddingBo.setFids(Arrays.asList("fid001", "fid002")); - -strategy.storeEmbeddings(storeEmbeddingBo); -``` - -### 3. 查询向量数据 - -```java -QueryVectorBo queryVectorBo = new QueryVectorBo(); -queryVectorBo.setQuery("查询文本"); -queryVectorBo.setKid("test001"); -queryVectorBo.setMaxResults(5); - -List results = strategy.getQueryVector(queryVectorBo); -``` - -### 4. 删除数据 - -```java -// 按文档ID删除 -strategy.removeByDocId("doc001", "test001"); - -// 按片段ID删除 -strategy.removeByFid("fid001", "test001"); - -// 删除整个集合 -strategy.removeById("test001", "model"); -``` - -## 部署要求 - -### Milvus服务部署 - -1. **Docker部署** (推荐) -```bash -# 下载docker-compose文件 -wget https://github.com/milvus-io/milvus/releases/download/v2.6.4/milvus-standalone-docker-compose.yml -O docker-compose.yml - -# 启动Milvus -docker-compose up -d -``` - -2. **验证部署** -```bash -# 检查服务状态 -docker-compose ps - -# 查看日志 -docker-compose logs milvus-standalone -``` - -### 系统要求 - -- **内存**: 最少8GB，推荐16GB+ -- **存储**: SSD推荐，至少50GB可用空间 -- **CPU**: 4核心以上 -- **网络**: 确保19530端口可访问 - -## 性能优化 - -### 1. 索引优化 - -根据数据量调整索引参数： -- 小数据集(<100万): nlist=1024 -- 中等数据集(100万-1000万): nlist=4096 -- 大数据集(>1000万): nlist=16384 - -### 2. 批量操作 - -- 批量插入：建议每批1000-10000条记录 -- 批量查询：避免频繁的单条查询 - -### 3. 内存管理 - -```yaml -# docker-compose.yml中的内存配置 -environment: - MILVUS_CONFIG_PATH: /milvus/configs/milvus.yaml -volumes: - - ./milvus.yaml:/milvus/configs/milvus.yaml -``` - -## 故障排除 - -### 常见问题 - -1. **连接失败** - - 检查Milvus服务是否启动 - - 验证网络连接和端口 - - 确认配置中的URL正确 - -2. **集合创建失败** - - 检查集合名称是否符合规范 - - 验证字段定义是否正确 - - 查看Milvus日志获取详细错误 - -3. **插入数据失败** - - 确认向量维度与schema一致 - - 检查数据格式是否正确 - - 验证集合是否已加载 - -4. **查询无结果** - - 确认集合中有数据 - - 检查查询参数设置 - - 验证向量化模型一致性 - -### 日志调试 - -启用详细日志： -```properties -logging.level.org.ruoyi.service.strategy.impl.MilvusVectorStoreStrategy=DEBUG -logging.level.io.milvus=DEBUG -``` - -## 与Weaviate对比 - -| 特性 | Milvus | Weaviate | -|------|--------|----------| -| 性能 | 高性能，专为大规模设计 | 中等性能 | -| 部署 | 需要独立部署 | 可独立部署或云服务 | -| 生态 | 专注向量搜索 | 集成更多AI功能 | -| 学习成本 | 中等 | 较低 | -| 扩展性 | 优秀 | 良好 | - -## 后续优化建议 - -1. **连接池管理**: 实现MilvusClient连接池 -2. **异步操作**: 支持异步插入和查询 -3. **分片策略**: 大数据集的分片管理 -4. **监控告警**: 集成性能监控 -5. **备份恢复**: 数据备份和恢复机制 - -## 参考资料 - -- [Milvus官方文档](https://milvus.io/docs) -- [Milvus Java SDK](https://github.com/milvus-io/milvus-sdk-java) -- [向量数据库最佳实践](https://milvus.io/docs/performance_faq.md) \ No newline at end of file diff --git a/ruoyi-admin/src/main/resources/application.yml b/ruoyi-admin/src/main/resources/application.yml index 6d5e6d2f..2bb9f120 100644 --- a/ruoyi-admin/src/main/resources/application.yml +++ b/ruoyi-admin/src/main/resources/application.yml @@ -328,3 +328,17 @@ spring: servers-configuration: classpath:mcp-server.json request-timeout: 300s +--- # 向量库配置 +vector-store: + # 向量存储类型 (weaviate/milvus) + type: weaviate + # Weaviate配置 + weaviate: + protocol: http + host: 127.0.0.1:6038 + classname: LocalKnowledge + # Milvus配置 + milvus: + url: http://localhost:19530 + collectionname: LocalKnowledge + diff --git a/ruoyi-common/ruoyi-common-core/src/main/java/org/ruoyi/common/core/config/VectorStoreProperties.java b/ruoyi-common/ruoyi-common-core/src/main/java/org/ruoyi/common/core/config/VectorStoreProperties.java new file mode 100644 index 00000000..98f3ddc4 --- /dev/null +++ b/ruoyi-common/ruoyi-common-core/src/main/java/org/ruoyi/common/core/config/VectorStoreProperties.java @@ -0,0 +1,62 @@ +package org.ruoyi.common.core.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * 向量库配置属性 + * + * @author ageer + */ +@Data +@Component +@ConfigurationProperties(prefix = "vector-store") +public class VectorStoreProperties { + + /** + * 向量库类型 + */ + private String type = "weaviate"; + + /** + * Weaviate配置 + */ + private Weaviate weaviate = new Weaviate(); + + /** + * Milvus配置 + */ + private Milvus milvus = new Milvus(); + + @Data + public static class Weaviate { + /** + * 协议 + */ + private String protocol = "http"; + + /** + * 主机地址 + */ + private String host = "localhost:8080"; + + /** + * 类名 + */ + private String classname = "Document"; + } + + @Data + public static class Milvus { + /** + * 连接URL + */ + private String url = "http://localhost:19530"; + + /** + * 集合名称 + */ + private String collectionname = "knowledge_base"; + } +} \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java index ae0a227d..4e78f6f3 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/VectorStoreService.java @@ -1,6 +1,6 @@ package org.ruoyi.service; -import com.google.protobuf.ServiceException; +import org.ruoyi.common.core.exception.ServiceException; import org.ruoyi.domain.bo.QueryVectorBo; import org.ruoyi.domain.bo.StoreEmbeddingBo; diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java index e7b023cc..677e4ce3 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java @@ -2,7 +2,6 @@ package org.ruoyi.service.impl; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.ruoyi.common.core.service.ConfigService; import org.ruoyi.domain.bo.QueryVectorBo; import org.ruoyi.domain.bo.StoreEmbeddingBo; import org.ruoyi.service.VectorStoreService; @@ -13,7 +12,7 @@ import org.springframework.stereotype.Service; import java.util.List; /** - * 向量库管理服务实现 - 使用策略模式 + * 向量库服务实现 * * @author ageer */ @@ -22,7 +21,6 @@ import java.util.List; @RequiredArgsConstructor public class VectorStoreServiceImpl implements VectorStoreService { - private final ConfigService configService; private final VectorStoreStrategyFactory strategyFactory; @@ -30,11 +28,7 @@ public class VectorStoreServiceImpl implements VectorStoreService { * 获取当前配置的向量库策略 */ private VectorStoreStrategy getCurrentStrategy() { - String vectorStoreType = configService.getConfigValue("vector", "type"); - if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) { - vectorStoreType = "weaviate"; // 默认使用weaviate - } - return strategyFactory.getStrategy(vectorStoreType); + return strategyFactory.getStrategy(); } @Override diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/AbstractVectorStoreStrategy.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/AbstractVectorStoreStrategy.java index 104714cb..7fdeb195 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/AbstractVectorStoreStrategy.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/AbstractVectorStoreStrategy.java @@ -1,13 +1,13 @@ package org.ruoyi.service.strategy; -import com.google.protobuf.ServiceException; +import org.ruoyi.common.core.exception.ServiceException; import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.ollama.OllamaEmbeddingModel; import dev.langchain4j.model.openai.OpenAiEmbeddingModel; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.ruoyi.common.core.service.ConfigService; +import org.ruoyi.common.core.config.VectorStoreProperties; /** * 向量库策略抽象基类 @@ -19,7 +19,7 @@ import org.ruoyi.common.core.service.ConfigService; @RequiredArgsConstructor public abstract class AbstractVectorStoreStrategy implements VectorStoreStrategy { - protected final ConfigService configService; + protected final VectorStoreProperties vectorStoreProperties; /** * 获取向量模型 diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategyFactory.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategyFactory.java index 1e606b22..fbd5b27b 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategyFactory.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/VectorStoreStrategyFactory.java @@ -1,15 +1,15 @@ package org.ruoyi.service.strategy; +import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.ruoyi.common.core.service.ConfigService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; +import org.ruoyi.common.core.config.VectorStoreProperties; +import org.ruoyi.service.strategy.impl.MilvusVectorStoreStrategy; +import org.ruoyi.service.strategy.impl.WeaviateVectorStoreStrategy; import org.springframework.stereotype.Component; +import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; /** * 向量库策略工厂 @@ -20,69 +20,55 @@ import java.util.concurrent.ConcurrentHashMap; @Slf4j @Component @RequiredArgsConstructor -public class VectorStoreStrategyFactory implements ApplicationContextAware { +public class VectorStoreStrategyFactory { - private final ConfigService configService; - private final Map strategyMap = new ConcurrentHashMap<>(); - private ApplicationContext applicationContext; + private final VectorStoreProperties vectorStoreProperties; + private final WeaviateVectorStoreStrategy weaviateStrategy; + private final MilvusVectorStoreStrategy milvusStrategy; - @Override - public void setApplicationContext(ApplicationContext applicationContext) { - this.applicationContext = applicationContext; - initStrategies(); - } + private Map strategies; - /** - * 初始化所有策略实现 - */ - private void initStrategies() { - Map strategies = applicationContext.getBeansOfType(VectorStoreStrategy.class); - for (VectorStoreStrategy strategy : strategies.values()) { - if (strategy instanceof AbstractVectorStoreStrategy) { - AbstractVectorStoreStrategy abstractStrategy = (AbstractVectorStoreStrategy) strategy; - strategyMap.put(abstractStrategy.getVectorStoreType(), strategy); - log.info("注册向量库策略: {}", abstractStrategy.getVectorStoreType()); - } - } + @PostConstruct + public void init() { + strategies = new HashMap<>(); + strategies.put("weaviate", weaviateStrategy); + strategies.put("milvus", milvusStrategy); + log.info("向量库策略工厂初始化完成，支持的策略: {}", strategies.keySet()); } /** * 获取当前配置的向量库策略 */ public VectorStoreStrategy getStrategy() { - String vectorStoreType = configService.getConfigValue("vector", "store_type"); - if (vectorStoreType == null || vectorStoreType.isEmpty()) { + String vectorStoreType = vectorStoreProperties.getType(); + if (vectorStoreType == null || vectorStoreType.trim().isEmpty()) { vectorStoreType = "weaviate"; // 默认使用weaviate } - VectorStoreStrategy strategy = strategyMap.get(vectorStoreType); + VectorStoreStrategy strategy = strategies.get(vectorStoreType.toLowerCase()); if (strategy == null) { log.warn("未找到向量库策略: {}, 使用默认策略: weaviate", vectorStoreType); - strategy = strategyMap.get("weaviate"); - } - - if (strategy == null) { - throw new RuntimeException("未找到可用的向量库策略实现"); + strategy = strategies.get("weaviate"); } + log.debug("使用向量库策略: {}", vectorStoreType); return strategy; } /** - * 根据类型获取特定的向量库策略 + * 根据类型获取向量库策略 */ - public VectorStoreStrategy getStrategy(String vectorStoreType) { - VectorStoreStrategy strategy = strategyMap.get(vectorStoreType); - if (strategy == null) { - throw new RuntimeException("未找到向量库策略: " + vectorStoreType); + public VectorStoreStrategy getStrategy(String type) { + if (type == null || type.trim().isEmpty()) { + return getStrategy(); } + + VectorStoreStrategy strategy = strategies.get(type.toLowerCase()); + if (strategy == null) { + log.warn("未找到向量库策略: {}, 使用默认策略", type); + return getStrategy(); + } + return strategy; } - - /** - * 获取所有可用的向量库类型 - */ - public String[] getAvailableTypes() { - return strategyMap.keySet().toArray(new String[0]); - } } \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java index 540eefc8..26a09f1f 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/MilvusVectorStoreStrategy.java @@ -1,30 +1,23 @@ package org.ruoyi.service.strategy.impl; -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.google.protobuf.ServiceException; +import org.ruoyi.common.core.exception.ServiceException; import dev.langchain4j.data.embedding.Embedding; import dev.langchain4j.model.embedding.EmbeddingModel; -import io.milvus.v2.client.ConnectConfig; -import io.milvus.v2.client.MilvusClientV2; -import io.milvus.v2.common.DataType; -import io.milvus.v2.common.IndexParam; -import io.milvus.v2.service.collection.request.AddFieldReq; -import io.milvus.v2.service.collection.request.CreateCollectionReq; -import io.milvus.v2.service.collection.request.DescribeCollectionReq; -import io.milvus.v2.service.collection.request.DropCollectionReq; -import io.milvus.v2.service.collection.request.HasCollectionReq; -import io.milvus.v2.service.collection.response.DescribeCollectionResp; -import io.milvus.v2.service.vector.request.DeleteReq; -import io.milvus.v2.service.vector.request.InsertReq; -import io.milvus.v2.service.vector.request.SearchReq; -import io.milvus.v2.service.vector.request.data.BaseVector; -import io.milvus.v2.service.vector.request.data.FloatVec; -import io.milvus.v2.service.vector.response.DeleteResp; -import io.milvus.v2.service.vector.response.InsertResp; -import io.milvus.v2.service.vector.response.SearchResp; +import io.milvus.client.MilvusServiceClient; +import io.milvus.common.clientenum.ConsistencyLevelEnum; +import io.milvus.grpc.*; +import io.milvus.param.*; +import io.milvus.param.collection.*; +import io.milvus.param.dml.DeleteParam; +import io.milvus.param.dml.InsertParam; +import io.milvus.param.dml.SearchParam; +import io.milvus.param.index.CreateIndexParam; +import io.milvus.param.index.DescribeIndexParam; +import io.milvus.response.DescCollResponseWrapper; +import io.milvus.response.SearchResultsWrapper; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.ruoyi.common.core.service.ConfigService; +import org.ruoyi.common.core.config.VectorStoreProperties; import org.ruoyi.domain.bo.QueryVectorBo; import org.ruoyi.domain.bo.StoreEmbeddingBo; import org.ruoyi.service.strategy.AbstractVectorStoreStrategy; @@ -41,105 +34,122 @@ import java.util.*; @Component public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy { - private MilvusClientV2 client; - - public MilvusVectorStoreStrategy(ConfigService configService) { - super(configService); + private MilvusServiceClient milvusClient; + + public MilvusVectorStoreStrategy(VectorStoreProperties vectorStoreProperties) { + super(vectorStoreProperties); } @Override public String getVectorStoreType() { return "milvus"; } - + @Override public void createSchema(String vectorModelName, String kid, String modelName) { - log.info("Milvus创建schema: vectorModelName={}, kid={}, modelName={}", vectorModelName, kid, modelName); + String url = vectorStoreProperties.getMilvus().getUrl(); + String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid; - // 1. 获取Milvus配置 - String host = configService.getConfigValue("milvus", "url"); - String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; - - ConnectConfig config = ConnectConfig.builder() - .uri(host) + // 创建Milvus客户端连接 + ConnectParam connectParam = ConnectParam.newBuilder() + .withUri(url) .build(); - client = new MilvusClientV2(config); + milvusClient = new MilvusServiceClient(connectParam); - // 2. 检查集合是否存在 - HasCollectionReq hasCollectionReq = HasCollectionReq.builder() - .collectionName(collectionName) + // 检查集合是否存在 + HasCollectionParam hasCollectionParam = HasCollectionParam.newBuilder() + .withCollectionName(collectionName) .build(); - Boolean hasCollection = client.hasCollection(hasCollectionReq); + R hasCollectionResponse = milvusClient.hasCollection(hasCollectionParam); + if (hasCollectionResponse.getStatus() != R.Status.Success.getCode()) { + log.error("检查集合是否存在失败: {}", hasCollectionResponse.getMessage()); + return; + } - if (!hasCollection) { - // 3. 创建集合schema - CreateCollectionReq.CollectionSchema schema = CreateCollectionReq.CollectionSchema.builder() - .build(); + if (!hasCollectionResponse.getData()) { + // 创建字段 + List fields = new ArrayList<>(); - // 添加字段定义 - schema.addField(AddFieldReq.builder() - .fieldName("id") - .dataType(DataType.Int64) - .isPrimaryKey(true) - .autoID(true) + // ID字段 (主键) + fields.add(FieldType.newBuilder() + .withName("id") + .withDataType(DataType.Int64) + .withPrimaryKey(true) + .withAutoID(true) + .build()); + + // 文本字段 + fields.add(FieldType.newBuilder() + .withName("text") + .withDataType(DataType.VarChar) + .withMaxLength(65535) + .build()); + + // fid字段 + fields.add(FieldType.newBuilder() + .withName("fid") + .withDataType(DataType.VarChar) + .withMaxLength(255) + .build()); + + // kid字段 + fields.add(FieldType.newBuilder() + .withName("kid") + .withDataType(DataType.VarChar) + .withMaxLength(255) + .build()); + + // docId字段 + fields.add(FieldType.newBuilder() + .withName("docId") + .withDataType(DataType.VarChar) + .withMaxLength(255) + .build()); + + // 向量字段 + fields.add(FieldType.newBuilder() + .withName("vector") + .withDataType(DataType.FloatVector) + .withDimension(1024) // 根据实际embedding维度调整 .build()); - schema.addField(AddFieldReq.builder() - .fieldName("text") - .dataType(DataType.VarChar) - .maxLength(65535) - .build()); - - schema.addField(AddFieldReq.builder() - .fieldName("fid") - .dataType(DataType.VarChar) - .maxLength(255) - .build()); - - schema.addField(AddFieldReq.builder() - .fieldName("kid") - .dataType(DataType.VarChar) - .maxLength(255) - .build()); - - schema.addField(AddFieldReq.builder() - .fieldName("docId") - .dataType(DataType.VarChar) - .maxLength(255) - .build()); - - schema.addField(AddFieldReq.builder() - .fieldName("vector") - .dataType(DataType.FloatVector) - .dimension(1024) // 根据实际embedding维度调整 - .build()); - - // 4. 创建索引参数 - List indexParams = new ArrayList<>(); - indexParams.add(IndexParam.builder() - .fieldName("vector") - .indexType(IndexParam.IndexType.IVF_FLAT) - .metricType(IndexParam.MetricType.L2) - .extraParams(Map.of("nlist", 1024)) - .build()); - - // 5. 创建集合 - CreateCollectionReq createCollectionReq = CreateCollectionReq.builder() - .collectionName(collectionName) - .collectionSchema(schema) - .indexParams(indexParams) + // 创建集合 + CreateCollectionParam createCollectionParam = CreateCollectionParam.newBuilder() + .withCollectionName(collectionName) + .withDescription("Knowledge base collection for " + kid) + .withShardsNum(2) + .withFieldTypes(fields) .build(); - client.createCollection(createCollectionReq); - log.info("Milvus集合创建成功: {}", collectionName); + R createCollectionResponse = milvusClient.createCollection(createCollectionParam); + if (createCollectionResponse.getStatus() != R.Status.Success.getCode()) { + log.error("创建集合失败: {}", createCollectionResponse.getMessage()); + return; + } + + // 创建索引 + CreateIndexParam createIndexParam = CreateIndexParam.newBuilder() + .withCollectionName(collectionName) + .withFieldName("vector") + .withIndexType(IndexType.IVF_FLAT) + .withMetricType(MetricType.L2) + .withExtraParam("{\"nlist\":1024}") + .build(); + + R createIndexResponse = milvusClient.createIndex(createIndexParam); + if (createIndexResponse.getStatus() != R.Status.Success.getCode()) { + log.error("创建索引失败: {}", createIndexResponse.getMessage()); + } else { + log.info("Milvus集合和索引创建成功: {}", collectionName); + } } else { log.info("Milvus集合已存在: {}", collectionName); } } @Override - public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) throws ServiceException { + public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) { createSchema(storeEmbeddingBo.getVectorModelName(), storeEmbeddingBo.getKid(), storeEmbeddingBo.getVectorModelName()); EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), @@ -149,12 +159,13 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy { List fidList = storeEmbeddingBo.getFids(); String kid = storeEmbeddingBo.getKid(); String docId = storeEmbeddingBo.getDocId(); - String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; + String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid; log.info("Milvus向量存储条数记录: " + chunkList.size()); long startTime = System.currentTimeMillis(); // 准备批量插入数据 + List fields = new ArrayList<>(); List textList = new ArrayList<>(); List fidListData = new ArrayList<>(); List kidList = new ArrayList<>(); @@ -178,31 +189,25 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy { vectorList.add(vector); } - // 构建插入数据 - List data = new ArrayList<>(); - Gson gson = new Gson(); - for (int i = 0; i < textList.size(); i++) { - JsonObject row = new JsonObject(); - row.addProperty("text", textList.get(i)); - row.addProperty("fid", fidListData.get(i)); - row.addProperty("kid", kidList.get(i)); - row.addProperty("docId", docIdList.get(i)); - row.add("vector", gson.toJsonTree(vectorList.get(i))); - data.add(row); - } + // 构建字段数据 + fields.add(new InsertParam.Field("text", textList)); + fields.add(new InsertParam.Field("fid", fidListData)); + fields.add(new InsertParam.Field("kid", kidList)); + fields.add(new InsertParam.Field("docId", docIdList)); + fields.add(new InsertParam.Field("vector", vectorList)); // 执行插入 - InsertReq insertReq = InsertReq.builder() - .collectionName(collectionName) - .data(data) + InsertParam insertParam = InsertParam.newBuilder() + .withCollectionName(collectionName) + .withFields(fields) .build(); - InsertResp insertResp = client.insert(insertReq); - if (insertResp.getInsertCnt() > 0) { - log.info("Milvus向量存储成功，插入条数: {}", insertResp.getInsertCnt()); - } else { - log.error("Milvus向量存储失败"); + R insertResponse = milvusClient.insert(insertParam); + if (insertResponse.getStatus() != R.Status.Success.getCode()) { + log.error("Milvus向量存储失败: {}", insertResponse.getMessage()); throw new ServiceException("Milvus向量存储失败"); + } else { + log.info("Milvus向量存储成功，插入条数: {}", insertResponse.getData().getInsertCnt()); } long endTime = System.currentTimeMillis(); @@ -217,99 +222,116 @@ public class MilvusVectorStoreStrategy extends AbstractVectorStoreStrategy { queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl()); Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content(); - String collectionName = configService.getConfigValue("milvus", "collectionname") + queryVectorBo.getKid(); + String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + queryVectorBo.getKid(); List resultList = new ArrayList<>(); - // 准备查询向量 - List searchVectors = new ArrayList<>(); - float[] queryVectorArray = new float[queryEmbedding.vector().length]; - for (int i = 0; i < queryEmbedding.vector().length; i++) { - queryVectorArray[i] = queryEmbedding.vector()[i]; - } - searchVectors.add(new FloatVec(queryVectorArray)); + // 加载集合到内存 + LoadCollectionParam loadCollectionParam = LoadCollectionParam.newBuilder() + .withCollectionName(collectionName) + .build(); + milvusClient.loadCollection(loadCollectionParam); - // 构建搜索请求 - SearchReq searchReq = SearchReq.builder() - .collectionName(collectionName) - .data(searchVectors) - .topK(queryVectorBo.getMaxResults()) - .outputFields(Arrays.asList("text", "fid", "kid", "docId")) + // 准备查询向量 + List> searchVectors = new ArrayList<>(); + List queryVector = new ArrayList<>(); + for (float f : queryEmbedding.vector()) { + queryVector.add(f); + } + searchVectors.add(queryVector); + + // 构建搜索参数 + SearchParam searchParam = SearchParam.newBuilder() + .withCollectionName(collectionName) + .withMetricType(MetricType.L2) + .withOutFields(Arrays.asList("text", "fid", "kid", "docId")) + .withTopK(queryVectorBo.getMaxResults()) + .withVectors(searchVectors) + .withVectorFieldName("vector") + .withParams("{\"nprobe\":10}") .build(); - SearchResp searchResp = client.search(searchReq); - if (searchResp != null && searchResp.getSearchResults() != null) { - List> searchResults = searchResp.getSearchResults(); + R searchResponse = milvusClient.search(searchParam); + if (searchResponse.getStatus() != R.Status.Success.getCode()) { + log.error("Milvus查询失败: {}", searchResponse.getMessage()); + return resultList; + } + + SearchResultsWrapper wrapper = new SearchResultsWrapper(searchResponse.getData().getResults()); + + // 遍历搜索结果 + for (int i = 0; i < wrapper.getIDScore(0).size(); i++) { + SearchResultsWrapper.IDScore idScore = wrapper.getIDScore(0).get(i); - for (List results : searchResults) { - for (SearchResp.SearchResult result : results) { - Map entity = result.getEntity(); - String text = (String) entity.get("text"); - if (text != null) { - resultList.add(text); - } + // 获取text字段数据 + List textFieldData = wrapper.getFieldData("text", 0); + if (textFieldData != null && i < textFieldData.size()) { + Object textObj = textFieldData.get(i); + if (textObj != null) { + resultList.add(textObj.toString()); + log.debug("找到相似文本，ID: {}, 距离: {}, 内容: {}", + idScore.getLongID(), idScore.getScore(), textObj.toString()); } } - } else { - log.error("Milvus查询失败或无结果"); } return resultList; } @Override - public void removeById(String id, String modelName) throws ServiceException { - String collectionName = configService.getConfigValue("milvus", "collectionname") + id; + @SneakyThrows + public void removeById(String id, String modelName) { + String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + id; // 删除整个集合 - DropCollectionReq dropCollectionReq = DropCollectionReq.builder() - .collectionName(collectionName) + DropCollectionParam dropCollectionParam = DropCollectionParam.newBuilder() + .withCollectionName(collectionName) .build(); - try { - client.dropCollection(dropCollectionReq); + R dropResponse = milvusClient.dropCollection(dropCollectionParam); + if (dropResponse.getStatus() != R.Status.Success.getCode()) { + log.error("Milvus集合删除失败: {}", dropResponse.getMessage()); + throw new ServiceException("Milvus集合删除失败"); + } else { log.info("Milvus集合删除成功: {}", collectionName); - } catch (Exception e) { - log.error("Milvus集合删除失败: {}", e.getMessage()); - throw new ServiceException("Milvus集合删除失败: " + e.getMessage()); } } @Override - public void removeByDocId(String docId, String kid) throws ServiceException { - String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; + public void removeByDocId(String docId, String kid) { + String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid; String expr = "docId == \"" + docId + "\""; - DeleteReq deleteReq = DeleteReq.builder() - .collectionName(collectionName) - .filter(expr) + DeleteParam deleteParam = DeleteParam.newBuilder() + .withCollectionName(collectionName) + .withExpr(expr) .build(); - try { - DeleteResp deleteResp = client.delete(deleteReq); - log.info("Milvus成功删除 docId={} 的所有向量数据，删除条数: {}", docId, deleteResp.getDeleteCnt()); - } catch (Exception e) { - log.error("Milvus删除失败: {}", e.getMessage()); - throw new ServiceException(e.getMessage()); + R deleteResponse = milvusClient.delete(deleteParam); + if (deleteResponse.getStatus() != R.Status.Success.getCode()) { + log.error("Milvus删除失败: {}", deleteResponse.getMessage()); + throw new ServiceException("Milvus删除失败"); + } else { + log.info("Milvus成功删除 docId={} 的所有向量数据，删除条数: {}", docId, deleteResponse.getData().getDeleteCnt()); } } @Override - public void removeByFid(String fid, String kid) throws ServiceException { - String collectionName = configService.getConfigValue("milvus", "collectionname") + kid; + public void removeByFid(String fid, String kid) { + String collectionName = vectorStoreProperties.getMilvus().getCollectionname() + kid; String expr = "fid == \"" + fid + "\""; - DeleteReq deleteReq = DeleteReq.builder() - .collectionName(collectionName) - .filter(expr) + DeleteParam deleteParam = DeleteParam.newBuilder() + .withCollectionName(collectionName) + .withExpr(expr) .build(); - try { - DeleteResp deleteResp = client.delete(deleteReq); - log.info("Milvus成功删除 fid={} 的所有向量数据，删除条数: {}", fid, deleteResp.getDeleteCnt()); - } catch (Exception e) { - log.error("Milvus删除失败: {}", e.getMessage()); - throw new ServiceException(e.getMessage()); + R deleteResponse = milvusClient.delete(deleteParam); + if (deleteResponse.getStatus() != R.Status.Success.getCode()) { + log.error("Milvus删除失败: {}", deleteResponse.getMessage()); + throw new ServiceException("Milvus删除失败"); + } else { + log.info("Milvus成功删除 fid={} 的所有向量数据，删除条数: {}", fid, deleteResponse.getData().getDeleteCnt()); } } } \ No newline at end of file diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/WeaviateVectorStoreStrategy.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/WeaviateVectorStoreStrategy.java index 9e3d3aec..6275d939 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/WeaviateVectorStoreStrategy.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/strategy/impl/WeaviateVectorStoreStrategy.java @@ -1,7 +1,7 @@ package org.ruoyi.service.strategy.impl; import cn.hutool.json.JSONObject; -import com.google.protobuf.ServiceException; +import org.ruoyi.common.core.exception.ServiceException; import dev.langchain4j.data.embedding.Embedding; import dev.langchain4j.model.embedding.EmbeddingModel; import io.weaviate.client.Config; @@ -17,7 +17,7 @@ import io.weaviate.client.v1.schema.model.Schema; import io.weaviate.client.v1.schema.model.WeaviateClass; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.ruoyi.common.core.service.ConfigService; +import org.ruoyi.common.core.config.VectorStoreProperties; import org.ruoyi.domain.bo.QueryVectorBo; import org.ruoyi.domain.bo.StoreEmbeddingBo; import org.ruoyi.service.strategy.AbstractVectorStoreStrategy; @@ -35,8 +35,8 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy { private WeaviateClient client; - public WeaviateVectorStoreStrategy(ConfigService configService) { - super(configService); + public WeaviateVectorStoreStrategy(VectorStoreProperties vectorStoreProperties) { + super(vectorStoreProperties); } @Override @@ -46,9 +46,9 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy { @Override public void createSchema(String vectorModelName, String kid, String modelName) { - String protocol = configService.getConfigValue("weaviate", "protocol"); - String host = configService.getConfigValue("weaviate", "host"); - String className = configService.getConfigValue("weaviate", "classname") + kid; + String protocol = vectorStoreProperties.getWeaviate().getProtocol(); + String host = vectorStoreProperties.getWeaviate().getHost(); + String className = vectorStoreProperties.getWeaviate().getClassname() + kid; // 创建 Weaviate 客户端 client = new WeaviateClient(new Config(protocol, host)); // 检查类是否存在，如果不存在就创建 schema @@ -128,7 +128,7 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy { vectorStrings.add(String.valueOf(v)); } String vectorStr = String.join(",", vectorStrings); - String className = configService.getConfigValue("weaviate", "classname"); + String className = vectorStoreProperties.getWeaviate().getClassname(); // 构建 GraphQL 查询 String graphQLQuery = String.format( @@ -176,9 +176,9 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy { @Override @SneakyThrows public void removeById(String id, String modelName) { - String protocol = configService.getConfigValue("weaviate", "protocol"); - String host = configService.getConfigValue("weaviate", "host"); - String className = configService.getConfigValue("weaviate", "classname"); + String protocol = vectorStoreProperties.getWeaviate().getProtocol(); + String host = vectorStoreProperties.getWeaviate().getHost(); + String className = vectorStoreProperties.getWeaviate().getClassname(); String finalClassName = className + id; WeaviateClient client = new WeaviateClient(new Config(protocol, host)); Result result = client.schema().classDeleter().withClassName(finalClassName).run(); @@ -192,7 +192,7 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy { @Override public void removeByDocId(String docId, String kid) { - String className = configService.getConfigValue("weaviate", "classname") + kid; + String className = vectorStoreProperties.getWeaviate().getClassname() + kid; // 构建 Where 条件 WhereFilter whereFilter = WhereFilter.builder() .path("docId") @@ -212,7 +212,7 @@ public class WeaviateVectorStoreStrategy extends AbstractVectorStoreStrategy { @Override public void removeByFid(String fid, String kid) { - String className = configService.getConfigValue("weaviate", "classname") + kid; + String className = vectorStoreProperties.getWeaviate().getClassname() + kid; // 构建 Where 条件 WhereFilter whereFilter = WhereFilter.builder() .path("fid") From 17c52e904832942b3963afcb7b8bbc681013f48d Mon Sep 17 00:00:00 2001 From: Yzm Date: Mon, 29 Sep 2025 21:49:27 +0800 Subject: [PATCH 4/6] =?UTF-8?q?refactor(VectorStoreServiceImpl):=20?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0@Primary=E6=B3=A8=E8=A7=A3=E4=BB=A5=E6=8C=87?= =?UTF-8?q?=E5=AE=9A=E4=B8=BB=E8=A6=81=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在多个实现存在时，明确指定VectorStoreServiceImpl作为主要实现类 --- .../java/org/ruoyi/service/impl/VectorStoreServiceImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java index 677e4ce3..9ba4513d 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java @@ -7,6 +7,7 @@ import org.ruoyi.domain.bo.StoreEmbeddingBo; import org.ruoyi.service.VectorStoreService; import org.ruoyi.service.strategy.VectorStoreStrategy; import org.ruoyi.service.strategy.VectorStoreStrategyFactory; +import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import java.util.List; @@ -16,6 +17,7 @@ import java.util.List; * * @author ageer */ +@Primary @Service @Slf4j @RequiredArgsConstructor From 77f7ac0af1e086e9ffce3e25476c96920923c72d Mon Sep 17 00:00:00 2001 From: Yzm Date: Sat, 11 Oct 2025 20:09:15 +0800 Subject: [PATCH 5/6] =?UTF-8?q?refactor(knowledge):=20=E6=A0=87=E8=AE=B0?= =?UTF-8?q?=E5=90=91=E9=87=8F=E5=AD=98=E5=82=A8=E6=9C=8D=E5=8A=A1=E4=B8=BA?= =?UTF-8?q?=E9=A6=96=E9=80=89=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 添加 @Primary 注解以指定为主要 Bean 实现 - 确保在多个实现存在时优先使用该服务 --- .../main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java index bbe5c8af..58b44f25 100644 --- a/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java +++ b/ruoyi-modules-api/ruoyi-knowledge-api/src/main/java/org/ruoyi/service/impl/VectorStoreServiceImpl.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; * @author ageer */ @Service +@Primary @Slf4j @RequiredArgsConstructor public class VectorStoreServiceImpl implements VectorStoreService { From 72337563ea0965e053c3f6a27b7d9d65a17f4847 Mon Sep 17 00:00:00 2001 From: Yzm Date: Sun, 12 Oct 2025 18:15:11 +0800 Subject: [PATCH 6/6] =?UTF-8?q?feat(chat):=20=E6=B7=BB=E5=8A=A0=E6=A0=B9?= =?UTF-8?q?=E6=8D=AE=E4=BC=9A=E8=AF=9DID=E6=9F=A5=E8=AF=A2=E8=81=8A?= =?UTF-8?q?=E5=A4=A9=E6=B6=88=E6=81=AF=E5=88=97=E8=A1=A8=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E4=BC=9A=E8=AF=9DID=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/application-dev.yml | 4 ++-- .../chat/ChatMessageController.java | 12 ++++++++++++ .../service/chat/impl/SseServiceImpl.java | 19 +------------------ 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/ruoyi-admin/src/main/resources/application-dev.yml b/ruoyi-admin/src/main/resources/application-dev.yml index c6e4e3f0..1fd51a25 100644 --- a/ruoyi-admin/src/main/resources/application-dev.yml +++ b/ruoyi-admin/src/main/resources/application-dev.yml @@ -16,9 +16,9 @@ spring: master: type: ${spring.datasource.type} driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/ruoyi-ai-github?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true + url: jdbc:mysql://127.0.0.1:3306/ruoyi-ai?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true username: root - password: qxyg1010 + password: root hikari: # 最大连接池数量 diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/controller/chat/ChatMessageController.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/controller/chat/ChatMessageController.java index 10ddd3c6..40a5971e 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/controller/chat/ChatMessageController.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/controller/chat/ChatMessageController.java @@ -45,6 +45,18 @@ public class ChatMessageController extends BaseController { return chatMessageService.queryPageList(bo, pageQuery); } + /** + * 根据会话ID查询聊天消息列表 + */ + @GetMapping("/listBySession/{sessionId}") + public TableDataInfo listBySession(@NotNull(message = "会话ID不能为空") + @PathVariable Long sessionId, + PageQuery pageQuery) { + ChatMessageBo bo = new ChatMessageBo(); + bo.setSessionId(sessionId); + return chatMessageService.queryPageList(bo, pageQuery); + } + /** * 导出聊天消息列表 */ diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java index 0250177a..e9a38259 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java @@ -100,24 +100,7 @@ public class SseServiceImpl implements ISseService { // 设置用户id chatRequest.setUserId(LoginHelper.getUserId()); - - //待优化的地方 （这里请前端提交send的时候传递uuid进来或者sessionId） - //待优化的地方 （这里请前端提交send的时候传递uuid进来或者sessionId） - //待优化的地方 （这里请前端提交send的时候传递uuid进来或者sessionId） - { - // 设置会话id - if (chatRequest.getUuid() == null) { - //暂时随机生成会话id - chatRequest.setSessionId(System.currentTimeMillis()); - } else { - //这里或许需要修改一下，这里应该用uuid 或者 前端传递 sessionId - chatRequest.setSessionId(chatRequest.getUuid()); - } - } - - - - chatRequest.setUserId(chatCostService.getUserId()); + // 设置会话id if (chatRequest.getSessionId() == null) { ChatSessionBo chatSessionBo = new ChatSessionBo(); chatSessionBo.setUserId(chatCostService.getUserId());