3 Commits

Author SHA1 Message Date
ageer
57b2f833f8 feat: 修复向量库初始化失败导致程序启动失败 2025-05-08 23:56:40 +08:00
ageer
7907cbeb7f feat: 修复向量库初始化失败导致程序启动失败 2025-05-08 19:31:46 +08:00
ageer
9cfcdd2b9b feat: 代码优化 2025-05-08 19:00:48 +08:00

View File

@@ -14,7 +14,6 @@ import dev.langchain4j.store.embedding.filter.comparison.IsEqualTo;
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore; import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
import dev.langchain4j.store.embedding.qdrant.QdrantEmbeddingStore; import dev.langchain4j.store.embedding.qdrant.QdrantEmbeddingStore;
import dev.langchain4j.store.embedding.weaviate.WeaviateEmbeddingStore; import dev.langchain4j.store.embedding.weaviate.WeaviateEmbeddingStore;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.service.ConfigService; import org.ruoyi.common.core.service.ConfigService;
@@ -29,8 +28,6 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
/** /**
* 向量库管理 * 向量库管理
* @author ageer * @author ageer
@@ -40,48 +37,54 @@ import java.util.Map;
@RequiredArgsConstructor @RequiredArgsConstructor
public class VectorStoreServiceImpl implements VectorStoreService { public class VectorStoreServiceImpl implements VectorStoreService {
private EmbeddingStore<TextSegment> embeddingStore;
private final ConfigService configService; private final ConfigService configService;
Map<String,EmbeddingStore<TextSegment>> storeMap;
@Override @Override
@PostConstruct
public void createSchema(String kid,String modelName) { public void createSchema(String kid,String modelName) {
if(modelName.equals("weaviate")){ EmbeddingStore<TextSegment> embeddingStore = WeaviateEmbeddingStore.builder().build();
String protocol = configService.getConfigValue("weaviate", "protocol"); switch (modelName) {
String host = configService.getConfigValue("weaviate", "host"); case "weaviate" -> {
String className = configService.getConfigValue("weaviate", "classname"); String protocol = configService.getConfigValue("weaviate", "protocol");
this.embeddingStore = WeaviateEmbeddingStore.builder() String host = configService.getConfigValue("weaviate", "host");
.scheme(protocol) String className = configService.getConfigValue("weaviate", "classname");
.host(host) embeddingStore = WeaviateEmbeddingStore.builder()
.objectClass(className+kid) .scheme(protocol)
.scheme(protocol) .host(host)
.avoidDups(true) .objectClass(className + kid)
.consistencyLevel("ALL") .scheme(protocol)
.build(); .avoidDups(true)
}else if(modelName.equals("milvus")){ .consistencyLevel("ALL")
String uri = configService.getConfigValue("milvus", "host"); .build();
String collection = configService.getConfigValue("milvus", "collection"); }
String dimension = configService.getConfigValue("milvus", "dimension"); case "milvus" -> {
this.embeddingStore = MilvusEmbeddingStore.builder() String uri = configService.getConfigValue("milvus", "host");
.uri(uri) String collection = configService.getConfigValue("milvus", "collection");
.collectionName(collection+kid) String dimension = configService.getConfigValue("milvus", "dimension");
.dimension(Integer.parseInt(dimension)) embeddingStore = MilvusEmbeddingStore.builder()
.build(); .uri(uri)
}else if(modelName.equals("qdrant")){ .collectionName(collection + kid)
String host = configService.getConfigValue("qdrant", "host"); .dimension(Integer.parseInt(dimension))
String port = configService.getConfigValue("qdrant", "port"); .build();
String collectionName = configService.getConfigValue("qdrant", "collectionName"); }
this.embeddingStore = QdrantEmbeddingStore.builder() case "qdrant" -> {
.host(host) String host = configService.getConfigValue("qdrant", "host");
.port(Integer.parseInt(port)) String port = configService.getConfigValue("qdrant", "port");
.collectionName(collectionName) String collectionName = configService.getConfigValue("qdrant", "collectionName");
.build(); embeddingStore = QdrantEmbeddingStore.builder()
.host(host)
.port(Integer.parseInt(port))
.collectionName(collectionName)
.build();
}
} }
storeMap.put(kid,embeddingStore);
} }
@Override @Override
public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) { public void storeEmbeddings(StoreEmbeddingBo storeEmbeddingBo) {
EmbeddingStore<TextSegment> store = storeMap.get(storeEmbeddingBo.getKid());
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getModelName(), EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getModelName(),
storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl()); storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl());
for (int i = 0; i < storeEmbeddingBo.getChunkList().size(); i++) { for (int i = 0; i < storeEmbeddingBo.getChunkList().size(); i++) {
@@ -93,12 +96,15 @@ public class VectorStoreServiceImpl implements VectorStoreService {
Embedding embedding = response.content(); Embedding embedding = response.content();
TextSegment segment = TextSegment.from(storeEmbeddingBo.getChunkList().get(i)); TextSegment segment = TextSegment.from(storeEmbeddingBo.getChunkList().get(i));
segment.metadata().putAll(dataSchema); segment.metadata().putAll(dataSchema);
embeddingStore.add(embedding,segment);
store.add(embedding,segment);
} }
} }
@Override @Override
public List<String> getQueryVector(QueryVectorBo queryVectorBo) { public List<String> getQueryVector(QueryVectorBo queryVectorBo) {
EmbeddingStore<TextSegment> store = storeMap.get(queryVectorBo.getKid());
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getModelName(), EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getModelName(),
queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl()); queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl());
Filter simpleFilter = new IsEqualTo("kid", queryVectorBo.getKid()); Filter simpleFilter = new IsEqualTo("kid", queryVectorBo.getKid());
@@ -109,38 +115,40 @@ public class VectorStoreServiceImpl implements VectorStoreService {
// 添加过滤条件 // 添加过滤条件
.filter(simpleFilter) .filter(simpleFilter)
.build(); .build();
List<EmbeddingMatch<TextSegment>> matches = embeddingStore.search(embeddingSearchRequest).matches(); List<EmbeddingMatch<TextSegment>> matches = store.search(embeddingSearchRequest).matches();
List<String> results = new ArrayList<>(); List<String> results = new ArrayList<>();
matches.forEach(embeddingMatch -> { matches.forEach(embeddingMatch -> results.add(embeddingMatch.embedded().text()));
results.add(embeddingMatch.embedded().text());
});
return results; return results;
} }
@Override @Override
public void removeByKid(String kid) { public void removeByKid(String kid) {
EmbeddingStore<TextSegment> store = storeMap.get(kid);
// 根据条件删除向量数据 // 根据条件删除向量数据
Filter simpleFilter = new IsEqualTo("kid", kid); Filter simpleFilter = new IsEqualTo("kid", kid);
embeddingStore.removeAll(simpleFilter); store.removeAll(simpleFilter);
} }
@Override @Override
public void removeByDocId(String kid, String docId) { public void removeByDocId(String kid, String docId) {
EmbeddingStore<TextSegment> store = storeMap.get(kid);
// 根据条件删除向量数据 // 根据条件删除向量数据
Filter simpleFilterByDocId = new IsEqualTo("docId", docId); Filter simpleFilterByDocId = new IsEqualTo("docId", docId);
embeddingStore.removeAll(simpleFilterByDocId); store.removeAll(simpleFilterByDocId);
} }
@Override @Override
public void removeByKidAndFid(String kid, String fid) { public void removeByKidAndFid(String kid, String fid) {
EmbeddingStore<TextSegment> store = storeMap.get(kid);
// 根据条件删除向量数据 // 根据条件删除向量数据
Filter simpleFilterByKid = new IsEqualTo("kid", kid); Filter simpleFilterByKid = new IsEqualTo("kid", kid);
Filter simpleFilterFid = new IsEqualTo("fid", fid); Filter simpleFilterFid = new IsEqualTo("fid", fid);
Filter simpleFilterByAnd = Filter.and(simpleFilterFid, simpleFilterByKid); Filter simpleFilterByAnd = Filter.and(simpleFilterFid, simpleFilterByKid);
embeddingStore.removeAll(simpleFilterByAnd); store.removeAll(simpleFilterByAnd);
} }
/** /**