feat: 向量库查询删除逻辑调整

This commit is contained in:
ageerle
2025-05-26 22:40:01 +08:00
parent 463ad6c583
commit 70ae7ea8f1
3 changed files with 18 additions and 55 deletions

View File

@@ -17,10 +17,7 @@ public interface VectorStoreService {
void createSchema(String kid,String modelName); void createSchema(String kid,String modelName);
void removeByKid(String kid,String modelName); void removeById(String id,String modelName);
void removeByDocId(String kid,String docId,String modelName);
void removeByKidAndFid(String kid, String fid,String modelName);
} }

View File

@@ -9,8 +9,6 @@ import dev.langchain4j.model.openai.OpenAiEmbeddingModel;
import dev.langchain4j.store.embedding.EmbeddingMatch; import dev.langchain4j.store.embedding.EmbeddingMatch;
import dev.langchain4j.store.embedding.EmbeddingSearchRequest; import dev.langchain4j.store.embedding.EmbeddingSearchRequest;
import dev.langchain4j.store.embedding.EmbeddingStore; import dev.langchain4j.store.embedding.EmbeddingStore;
import dev.langchain4j.store.embedding.filter.Filter;
import dev.langchain4j.store.embedding.filter.comparison.IsEqualTo;
import dev.langchain4j.store.embedding.weaviate.WeaviateEmbeddingStore; import dev.langchain4j.store.embedding.weaviate.WeaviateEmbeddingStore;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@@ -22,9 +20,7 @@ import org.ruoyi.service.VectorStoreService;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* 向量库管理 * 向量库管理
@@ -40,6 +36,7 @@ public class VectorStoreServiceImpl implements VectorStoreService {
private EmbeddingStore<TextSegment> embeddingStore; private EmbeddingStore<TextSegment> embeddingStore;
@Override @Override
public void createSchema(String kid, String modelName) { public void createSchema(String kid, String modelName) {
String protocol = configService.getConfigValue("weaviate", "protocol"); String protocol = configService.getConfigValue("weaviate", "protocol");
@@ -48,7 +45,7 @@ public class VectorStoreServiceImpl implements VectorStoreService {
embeddingStore = WeaviateEmbeddingStore.builder() embeddingStore = WeaviateEmbeddingStore.builder()
.scheme(protocol) .scheme(protocol)
.host(host) .host(host)
.objectClass(className + kid) .objectClass(className+kid)
.scheme(protocol) .scheme(protocol)
.avoidDups(true) .avoidDups(true)
.consistencyLevel("ALL") .consistencyLevel("ALL")
@@ -61,14 +58,9 @@ public class VectorStoreServiceImpl implements VectorStoreService {
EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(), EmbeddingModel embeddingModel = getEmbeddingModel(storeEmbeddingBo.getEmbeddingModelName(),
storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl()); storeEmbeddingBo.getApiKey(), storeEmbeddingBo.getBaseUrl());
List<String> chunkList = storeEmbeddingBo.getChunkList(); List<String> chunkList = storeEmbeddingBo.getChunkList();
for (int i = 0; i < chunkList.size(); i++) { for (String s : chunkList) {
Map<String, Object> dataSchema = new HashMap<>(); Embedding embedding = embeddingModel.embed(s).content();
dataSchema.put("kid", storeEmbeddingBo.getKid()); TextSegment segment = TextSegment.from(s);
dataSchema.put("docId", storeEmbeddingBo.getDocId());
dataSchema.put("fid", storeEmbeddingBo.getFids().get(i));
Embedding embedding = embeddingModel.embed(chunkList.get(i)).content();
TextSegment segment = TextSegment.from(chunkList.get(i));
segment.metadata().putAll(dataSchema);
embeddingStore.add(embedding, segment); embeddingStore.add(embedding, segment);
} }
} }
@@ -78,13 +70,10 @@ public class VectorStoreServiceImpl implements VectorStoreService {
createSchema(queryVectorBo.getKid(), queryVectorBo.getVectorModelName()); createSchema(queryVectorBo.getKid(), queryVectorBo.getVectorModelName());
EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(), EmbeddingModel embeddingModel = getEmbeddingModel(queryVectorBo.getEmbeddingModelName(),
queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl()); queryVectorBo.getApiKey(), queryVectorBo.getBaseUrl());
// Filter simpleFilter = new IsEqualTo("kid", queryVectorBo.getKid());
Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content(); Embedding queryEmbedding = embeddingModel.embed(queryVectorBo.getQuery()).content();
EmbeddingSearchRequest embeddingSearchRequest = EmbeddingSearchRequest.builder() EmbeddingSearchRequest embeddingSearchRequest = EmbeddingSearchRequest.builder()
.queryEmbedding(queryEmbedding) .queryEmbedding(queryEmbedding)
.maxResults(queryVectorBo.getMaxResults()) .maxResults(queryVectorBo.getMaxResults())
// 添加过滤条件
// .filter(simpleFilter)
.build(); .build();
List<EmbeddingMatch<TextSegment>> matches = embeddingStore.search(embeddingSearchRequest).matches(); List<EmbeddingMatch<TextSegment>> matches = embeddingStore.search(embeddingSearchRequest).matches();
List<String> results = new ArrayList<>(); List<String> results = new ArrayList<>();
@@ -94,29 +83,10 @@ public class VectorStoreServiceImpl implements VectorStoreService {
@Override @Override
public void removeByKid(String kid, String modelName) { public void removeById(String id, String modelName) {
createSchema(kid, modelName); createSchema(id, modelName);
// 根据条件删除向量数据 // 根据条件删除向量数据
Filter simpleFilter = new IsEqualTo("kid", kid); embeddingStore.remove(id);
embeddingStore.removeAll(simpleFilter);
}
@Override
public void removeByDocId(String kid, String docId, String modelName) {
createSchema(kid, modelName);
// 根据条件删除向量数据
Filter simpleFilterByDocId = new IsEqualTo("docId", docId);
embeddingStore.removeAll(simpleFilterByDocId);
}
@Override
public void removeByKidAndFid(String kid, String fid, String modelName) {
createSchema(kid, modelName);
// 根据条件删除向量数据
Filter simpleFilterByKid = new IsEqualTo("kid", kid);
Filter simpleFilterFid = new IsEqualTo("fid", fid);
Filter simpleFilterByAnd = Filter.and(simpleFilterFid, simpleFilterByKid);
embeddingStore.removeAll(simpleFilterByAnd);
} }
/** /**

View File

@@ -177,13 +177,11 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public void removeKnowledge(String id) { public void removeKnowledge(String id) {
Map<String,Object> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
map.put("kid",id); KnowledgeInfo knowledgeInfo = baseMapper.selectById(id);
List<KnowledgeInfoVo> knowledgeInfoList = baseMapper.selectVoByMap(map); check(knowledgeInfo);
check(knowledgeInfoList); map.put("kid",knowledgeInfo.getKid());
// 删除向量库信息 // 删除向量数据
// knowledgeInfoList.forEach(knowledgeInfoVo -> { vectorStoreService.removeById(String.valueOf(knowledgeInfo.getId()),knowledgeInfo.getVectorModelName());
// vectorStoreService.removeByKid(String.valueOf(knowledgeInfoVo.getId()),knowledgeInfoVo.getVectorModelName());
// });
// 删除附件和知识片段 // 删除附件和知识片段
fragmentMapper.deleteByMap(map); fragmentMapper.deleteByMap(map);
attachMapper.deleteByMap(map); attachMapper.deleteByMap(map);
@@ -256,14 +254,12 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
/** /**
* 检查用户是否有删除知识库权限 * 检查用户是否有删除知识库权限
* *
* @param knowledgeInfoList 知识库列表 * @param knowledgeInfo 知识库
*/ */
public void check(List<KnowledgeInfoVo> knowledgeInfoList) { public void check( KnowledgeInfo knowledgeInfo) {
LoginUser loginUser = LoginHelper.getLoginUser(); LoginUser loginUser = LoginHelper.getLoginUser();
for (KnowledgeInfoVo knowledgeInfoVo : knowledgeInfoList) { if (!knowledgeInfo.getUid().equals(loginUser.getUserId())) {
if (!knowledgeInfoVo.getUid().equals(loginUser.getUserId())) { throw new SecurityException("权限不足");
throw new SecurityException("权限不足");
}
} }
} }