xieyinlu/ruoyi-ai
1
0
Fork 0
mirror of https://gitcode.com/ageerle/ruoyi-ai.git synced 2026-03-26 19:13:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加知识库配置属性

Browse Source
This commit is contained in:
ageerle
2025-03-07 09:56:15 +08:00
parent d1b6a13a58
commit ea09421b0e
40 changed files with 332 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
.editorconfig
View File

@@ -1,18 +0,0 @@
# http://editorconfig.org
root = true
# 空格替代Tab缩进在各种编辑工具下效果一致
[*]
indent_style = space
indent_size = 4
charset = utf-8
end_of_line = lf
trim_trailing_whitespace = true
insert_final_newline = true
[*.{json,yml,yaml}]
indent_size = 2
[*.md]
insert_final_newline = false
trim_trailing_whitespace = false

2
ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java
View File

@@ -81,7 +81,7 @@ public class KnowledgeController extends BaseController {
List<Message> messages = chatRequest.getMessages();
String content = messages.get(messages.size() - 1).getContent().toString();
List<String> nearestList;
List<Double> queryVector = embeddingService.getQueryVector(content);
List<Double> queryVector = embeddingService.getQueryVector(content, chatRequest.getKid());
nearestList = vectorStore.nearest(queryVector,chatRequest.getKid());
for (String prompt : nearestList) {
Message sysMessage = Message.builder().content(prompt).role(Message.Role.USER).build();

2
ruoyi-admin/src/main/resources/application-dev.yml
View File

@@ -27,7 +27,7 @@ spring:
driverClassName: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://43.139.70.230:3306/ruoyi-ai?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true
username: ruoyi-ai
password: ruoyi-ai
password: eCaZ278N62k6fhYj
hikari:
# 最大连接池数量

30
ruoyi-admin/src/main/resources/application.yml
View File

@@ -310,11 +310,6 @@ wx:
# 企业微信应用
wechat:
# 是否使用微信 true/false
enable: true
# 生成的登录二维码路径 默认与项目同级
qrPath: "./"
# 企业微信应用
cp:
corpId:
appConfigs:
@@ -323,28 +318,5 @@ wechat:
token: ''
aesKey: ''
# 知识库配置
chain:
split:
chunk:
endspliter: "<STOP>"
# 分块文本大小
size: 200
overlay: 30
qaspliter: "###"
# 知识库中检索的条数
limits: 5
vector:
model: 'text-embedding-3-small'
store:
type: weaviate
weaviate:
protocol: http
host: 127.0.0.1:6038
classname: LocalKnowledge
milvus:
host: 127.0.0.1
port: 19530
dimension: 1536
collection: LocalKnowledge

4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java
View File

@@ -31,7 +31,7 @@ public class CodeFileLoader implements ResourceLoader{
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content){
return textSplitter.split(content);
public List<String> getChunkList(String content, String kid){
return textSplitter.split(content, kid);
}
}

2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java
View File

@@ -10,7 +10,7 @@ public class CsvFileLoader implements ResourceLoader{
}
@Override
public List<String> getChunkList(String content) {
public List<String> getChunkList(String content, String kid) {
return null;
}
}

2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java
View File

@@ -10,7 +10,7 @@ public class FolderLoader implements ResourceLoader{
}
@Override
public List<String> getChunkList(String content) {
public List<String> getChunkList(String content, String kid) {
return null;
}
}

2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java
View File

@@ -10,7 +10,7 @@ public class GithubLoader implements ResourceLoader{
}
@Override
public List<String> getChunkList(String content) {
public List<String> getChunkList(String content, String kid) {
return null;
}
}

2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java
View File

@@ -10,7 +10,7 @@ public class JsonFileLoader implements ResourceLoader{
}
@Override
public List<String> getChunkList(String content) {
public List<String> getChunkList(String content, String kid) {
return null;
}
}

4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java
View File

@@ -31,7 +31,7 @@ public class MarkDownFileLoader implements ResourceLoader{
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content){
return textSplitter.split(content);
public List<String> getChunkList(String content, String kid){
return textSplitter.split(content, kid);
}
}

4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java
View File

@@ -28,7 +28,7 @@ public class PdfFileLoader implements ResourceLoader{
}
@Override
public List<String> getChunkList(String content) {
return characterTextSplitter.split(content);
public List<String> getChunkList(String content, String kid) {
return characterTextSplitter.split(content, kid);
}
}

4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java
View File

@@ -7,6 +7,8 @@ import java.util.List;
* 资源载入
*/
public interface ResourceLoader {
String getContent(InputStream inputStream);
List<String> getChunkList(String content);
List<String> getChunkList(String content, String kid);
}

4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java
View File

@@ -31,7 +31,7 @@ public class TextFileLoader implements ResourceLoader{
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content){
return textSplitter.split(content);
public List<String> getChunkList(String content, String kid){
return textSplitter.split(content, kid);
}
}

4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java
View File

@@ -30,8 +30,8 @@ public class WordLoader implements ResourceLoader{
}
@Override
public List<String> getChunkList(String content) {
return textSplitter.split(content);
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
}
}

16
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/retrieve/PromptRetrieverProperties.java
View File

@@ -1,16 +0,0 @@
package org.ruoyi.knowledge.chain.retrieve;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Data
@Component
public class PromptRetrieverProperties {
/**
* 从知识库中检索的条数，limits 应大于 num
*/
@Value("${chain.limits}")
private int limits;
}

43
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java
View File

@@ -1,7 +1,10 @@
package org.ruoyi.knowledge.chain.split;
import lombok.AllArgsConstructor;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@@ -10,38 +13,46 @@ import java.util.Arrays;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
@Primary
public class CharacterTextSplitter implements TextSplitter{
private final SplitterProperties splitterProperties;
public class CharacterTextSplitter implements TextSplitter {
@Lazy
@Resource
private IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content) {
public List<String> split(String content, String kid) {
// 从知识库表中获取配置
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));
String knowledgeSeparator = knowledgeInfoVo.getKnowledgeSeparator();
int textBlockSize = knowledgeInfoVo.getTextBlockSize();
int overlapChar = knowledgeInfoVo.getOverlapChar();
List<String> chunkList = new ArrayList<>();
if (content.contains(splitterProperties.getEndspliter())){
if (content.contains(knowledgeSeparator)) {
// 按自定义分隔符切分
String[] chunks = content.split(splitterProperties.getEndspliter());
String[] chunks = content.split(knowledgeSeparator);
chunkList.addAll(Arrays.asList(chunks));
}else {
} else {
int indexMin = 0;
int len = content.length();
int i = 0;
int right = 0;
while (true) {
if (len > right ){
int begin = i*splitterProperties.getSize() - splitterProperties.getOverlay();
if (begin < indexMin){
if (len > right) {
int begin = i * textBlockSize - overlapChar;
if (begin < indexMin) {
begin = indexMin;
}
int end = splitterProperties.getSize()*(i+1) + splitterProperties.getOverlay();
if (end > len){
int end = textBlockSize * (i + 1) + overlapChar;
if (end > len) {
end = len;
}
String chunk = content.substring(begin,end);
String chunk = content.substring(begin, end);
chunkList.add(chunk);
i++;
right = right + splitterProperties.getSize();
}else {
right = right + textBlockSize;
} else {
break;
}
}

2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java
View File

@@ -11,7 +11,7 @@ import java.util.List;
@Slf4j
public class CodeTextSplitter implements TextSplitter{
@Override
public List<String> split(String content) {
public List<String> split(String content, String kid) {
return null;
}
}

2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java
View File

@@ -11,7 +11,7 @@ import java.util.List;
@Slf4j
public class MarkdownTextSplitter implements TextSplitter{
@Override
public List<String> split(String content) {
public List<String> split(String content, String kid) {
return null;
}
}

30
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/SplitterProperties.java
View File

@@ -1,30 +0,0 @@
package org.ruoyi.knowledge.chain.split;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Data
@Component
@ConfigurationProperties(prefix = "chain.split.chunk")
public class SplitterProperties {
/**
* 分段标识符
*/
private String endspliter;
/**
* 提问分段标识符
*/
private String qaspliter;
/**
* 分块文本大小
*/
private int size;
/**
* 相邻块之间重叠的字符数(避免边界信息丢失)
*/
private int overlay;
}

9
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java
View File

@@ -7,5 +7,12 @@ import java.util.List;
*/
public interface TextSplitter {
List<String> split(String content);
/**
* 文本切分
*
* @param content 文本内容
* @param kid 知识库id
* @return 切分后的文本列表
*/
List<String> split(String content, String kid);
}

2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java
View File

@@ -11,7 +11,7 @@ import java.util.List;
@Slf4j
public class TokenTextSplitter implements TextSplitter{
@Override
public List<String> split(String content) {
public List<String> split(String content, String kid) {
return null;
}
}

19
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java
View File

@@ -1,5 +1,6 @@
package org.ruoyi.knowledge.chain.vectorizer;
import jakarta.annotation.Resource;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -8,7 +9,10 @@ import org.ruoyi.common.chat.entity.embeddings.Embedding;
import org.ruoyi.common.chat.entity.embeddings.EmbeddingResponse;
import org.ruoyi.common.chat.openai.OpenAiStreamClient;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import java.math.BigDecimal;
@@ -20,8 +24,9 @@ import java.util.List;
@RequiredArgsConstructor
public class OpenAiVectorization implements Vectorization {
@Value("${chain.vector.model}")
private String embeddingModel;
@Lazy
@Resource
private IKnowledgeInfoService knowledgeInfoService;
@Getter
private OpenAiStreamClient openAiStreamClient;
@@ -29,12 +34,12 @@ public class OpenAiVectorization implements Vectorization {
private final ChatConfig chatConfig;
@Override
public List<List<Double>> batchVectorization(List<String> chunkList) {
public List<List<Double>> batchVectorization(List<String> chunkList, String kid) {
openAiStreamClient = chatConfig.getOpenAiStreamClient();
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));
Embedding embedding = Embedding.builder()
.input(chunkList)
.model(embeddingModel)
.model(knowledgeInfoVo.getVectorModel())
.build();
EmbeddingResponse embeddings = openAiStreamClient.embeddings(embedding);
List<List<Double>> vectorList = new ArrayList<>();
@@ -50,10 +55,10 @@ public class OpenAiVectorization implements Vectorization {
}
@Override
public List<Double> singleVectorization(String chunk) {
public List<Double> singleVectorization(String chunk, String kid) {
List<String> chunkList = new ArrayList<>();
chunkList.add(chunk);
List<List<Double>> vectorList = batchVectorization(chunkList);
List<List<Double>> vectorList = batchVectorization(chunkList, kid);
return vectorList.get(0);
}

5
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java
View File

@@ -6,6 +6,7 @@ import java.util.List;
* 向量化
*/
public interface Vectorization {
List<List<Double>> batchVectorization(List<String> chunkList);
List<Double> singleVectorization(String chunk);
List<List<Double>> batchVectorization(List<String> chunkList, String kid);
List<Double> singleVectorization(String chunk, String kid);
}

8
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java
View File

@@ -15,14 +15,14 @@ public class VectorizationWrapper implements Vectorization{
private final VectorizationFactory vectorizationFactory;
@Override
public List<List<Double>> batchVectorization(List<String> chunkList) {
public List<List<Double>> batchVectorization(List<String> chunkList, String kid) {
Vectorization embedding = vectorizationFactory.getEmbedding();
return embedding.batchVectorization(chunkList);
return embedding.batchVectorization(chunkList, kid);
}
@Override
public List<Double> singleVectorization(String chunk) {
public List<Double> singleVectorization(String chunk, String kid) {
Vectorization embedding = vectorizationFactory.getEmbedding();
return embedding.singleVectorization(chunk);
return embedding.singleVectorization(chunk, kid);
}
}

29
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java
View File

@@ -19,8 +19,10 @@ import io.milvus.param.partition.CreatePartitionParam;
import io.milvus.response.QueryResultsWrapper;
import io.milvus.response.SearchResultsWrapper;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.ruoyi.common.core.service.ConfigService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@@ -32,26 +34,27 @@ import java.util.List;
@Slf4j
public class MilvusVectorStore implements VectorStore{
@Value("${chain.vector.store.milvus.host}")
private String milvusHost;
@Value("${chain.vector.store.milvus.port}")
private Integer milvausPort;
@Value("${chain.vector.store.milvus.dimension}")
private Integer dimension;
@Value("${chain.vector.store.milvus.collection}")
private String collectionName;
private volatile Integer dimension;
private volatile String collectionName;
private MilvusServiceClient milvusServiceClient;
@Resource
private ConfigService configService;
@PostConstruct
public void loadConfig() {
this.dimension = Integer.parseInt(configService.getConfigValue("milvus", "dimension"));
this.collectionName = configService.getConfigValue("milvus", "collection");
}
@PostConstruct
public void init(){
String milvusHost = configService.getConfigValue("milvus", "host");
String milvausPort = configService.getConfigValue("milvus", "port");
milvusServiceClient = new MilvusServiceClient(
ConnectParam.newBuilder()
.withHost(milvusHost)
.withPort(milvausPort)
.withPort(Integer.parseInt(milvausPort))
.withDatabaseName("default")
.build()
);

30
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java
View File

@@ -1,29 +1,37 @@
package org.ruoyi.knowledge.chain.vectorstore;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.ruoyi.knowledge.domain.KnowledgeInfo;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
@Slf4j
@RequiredArgsConstructor
public class VectorStoreFactory {
@Value("${chain.vector.store.type}")
private String type;
private final WeaviateVectorStore weaviateVectorStore;
private final MilvusVectorStore milvusVectorStore;
public VectorStoreFactory(WeaviateVectorStore weaviateVectorStore, MilvusVectorStore milvusVectorStore) {
this.weaviateVectorStore = weaviateVectorStore;
this.milvusVectorStore = milvusVectorStore;
}
private final KnowledgeInfoMapper knowledgeInfoMapper;
public VectorStore getVectorStore(){
if ("weaviate".equals(type)){
public VectorStore getVectorStore(String kid){
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoMapper.selectVoOne(
new LambdaQueryWrapper<KnowledgeInfo>().eq(KnowledgeInfo::getKid,kid)
);
String vectorModel = knowledgeInfoVo.getVector();
if ("weaviate".equals(vectorModel)){
return weaviateVectorStore;
}else if ("milvus".equals(type)){
}else if ("milvus".equals(vectorModel)){
return milvusVectorStore;
}
return null;

14
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java
View File

@@ -16,43 +16,43 @@ public class VectorStoreWrapper implements VectorStore{
private final VectorStoreFactory vectorStoreFactory;
@Override
public void storeEmbeddings(List<String> chunkList, List<List<Double>> vectorList, String kid, String docId, List<String> fidList) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore();
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.storeEmbeddings(chunkList, vectorList, kid, docId, fidList);
}
@Override
public void removeByDocId(String kid, String docId) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore();
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.removeByDocId(kid,docId);
}
@Override
public void removeByKid(String kid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore();
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.removeByKid(kid);
}
@Override
public List<String> nearest(List<Double> queryVector, String kid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore();
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
return vectorStore.nearest(queryVector,kid);
}
@Override
public List<String> nearest(String query, String kid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore();
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
return vectorStore.nearest(query, kid);
}
@Override
public void newSchema(String kid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore();
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.newSchema(kid);
}
@Override
public void removeByKidAndFid(String kid, String fid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore();
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.removeByKidAndFid(kid, fid);
}
}

37
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java
View File

@@ -22,10 +22,15 @@ import io.weaviate.client.v1.schema.model.DataType;
import io.weaviate.client.v1.schema.model.Property;
import io.weaviate.client.v1.schema.model.Schema;
import io.weaviate.client.v1.schema.model.WeaviateClass;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.ruoyi.knowledge.chain.retrieve.PromptRetrieverProperties;
import org.ruoyi.common.core.service.ConfigService;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@@ -37,18 +42,23 @@ import java.util.Map;
@Slf4j
public class WeaviateVectorStore implements VectorStore{
@Value("${chain.vector.store.weaviate.protocol}")
private String protocol;
@Value("${chain.vector.store.weaviate.host}")
private String host;
private volatile String protocol;
private volatile String host;
private volatile String className;
@Value("${chain.vector.store.weaviate.classname}")
private String className;
@Lazy
@Resource
private IKnowledgeInfoService knowledgeInfoService;
private final PromptRetrieverProperties promptRetrieverProperties;
@Lazy
@Resource
private ConfigService configService;
public WeaviateVectorStore(PromptRetrieverProperties promptRetrieverProperties) {
this.promptRetrieverProperties = promptRetrieverProperties;
@PostConstruct
public void loadConfig() {
this.protocol = configService.getConfigValue("weaviate", "protocol");
this.host = configService.getConfigValue("weaviate", "host");
this.className = configService.getConfigValue("weaviate", "classname");
}
public WeaviateClient getClient(){
@@ -309,11 +319,12 @@ public class WeaviateVectorStore implements VectorStore{
.vector(vf)
.distance(1.6f) // certainty = 1f - distance /2f
.build();
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));
Result<GraphQLResponse> result = client.graphQL().get()
.withClassName(className + kid)
.withFields(contentField,_additional)
.withNearVector(nearVector)
.withLimit(promptRetrieverProperties.getLimits())
.withLimit(knowledgeInfoVo.getRetrieveLimit())
.run();
LinkedTreeMap<String,Object> t = (LinkedTreeMap<String, Object>) result.getResult().getData();
LinkedTreeMap<String,ArrayList<LinkedTreeMap>> l = (LinkedTreeMap<String, ArrayList<LinkedTreeMap>>) t.get("Get");
@@ -342,12 +353,12 @@ public class WeaviateVectorStore implements VectorStore{
.concepts(new String[]{ query })
.distance(1.6f) // certainty = 1f - distance /2f
.build();
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));
Result<GraphQLResponse> result = client.graphQL().get()
.withClassName(className + kid)
.withFields(contentField,_additional)
.withNearText(nearText)
.withLimit(promptRetrieverProperties.getLimits())
.withLimit(knowledgeInfoVo.getRetrieveLimit())
.run();
LinkedTreeMap<String,Object> t = (LinkedTreeMap<String, Object>) result.getResult().getData();
LinkedTreeMap<String,ArrayList<LinkedTreeMap>> l = (LinkedTreeMap<String, ArrayList<LinkedTreeMap>>) t.get("Get");

39
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java
View File

@@ -2,6 +2,7 @@ package org.ruoyi.knowledge.domain;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
import java.io.Serial;
@@ -43,6 +44,11 @@ public class KnowledgeInfo implements Serializable {
*/
private String kname;
/**
* 知识库名称
*/
private String share;
/**
* 描述
*/
@@ -59,5 +65,38 @@ public class KnowledgeInfo implements Serializable {
*/
private Date createTime;
/**
* 知识分隔符
*/
private String knowledgeSeparator;
/**
* 提问分隔符
*/
private String questionSeparator;
/**
* 重叠字符数
*/
private Integer overlapChar;
/**
* 知识库中检索的条数
*/
private Integer retrieveLimit;
/**
* 文本块大小
*/
private Integer textBlockSize;
/**
* 向量库
*/
private String vector;
/**
* 向量模型
*/
private String vectorModel;
}

46
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java
View File

@@ -43,11 +43,57 @@ public class KnowledgeInfoBo extends BaseEntity {
@NotBlank(message = "知识库名称不能为空")
private String kname;
/**
* 知识库名称
*/
@NotBlank(message = "是否公开知识库")
private String share;
/**
* 描述
*/
@NotBlank(message = "描述不能为空")
private String description;
/**
* 知识分隔符
*/
@NotBlank(message = "知识分隔符不能为空")
private String knowledgeSeparator;
/**
* 提问分隔符
*/
@NotBlank(message = "提问分隔符不能为空")
private String questionSeparator;
/**
* 重叠字符数
*/
@NotNull(message = "重叠字符数不能为空")
private Integer overlapChar;
/**
* 知识库中检索的条数
*/
@NotNull(message = "知识库中检索的条数不能为空")
private Integer retrieveLimit;
/**
* 文本块大小
*/
@NotNull(message = "文本块大小不能为空")
private Integer textBlockSize;
/**
* 向量库
*/
@NotBlank(message = "向量库不能为空")
private String vector;
/**
* 向量模型
*/
@NotBlank(message = "向量模型不能为空")
private String vectorModel;
}

45
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java
View File

@@ -47,11 +47,56 @@ public class KnowledgeInfoVo implements Serializable {
@ExcelProperty(value = "知识库名称")
private String kname;
/**
* 知识库名称
*/
private String share;
/**
* 描述
*/
@ExcelProperty(value = "描述")
private String description;
/**
* 知识分隔符
*/
@ExcelProperty(value = "知识分隔符")
private String knowledgeSeparator;
/**
* 提问分隔符
*/
@ExcelProperty(value = "提问分隔符")
private String questionSeparator;
/**
* 重叠字符数
*/
@ExcelProperty(value = "重叠字符数")
private Integer overlapChar;
/**
* 知识库中检索的条数
*/
@ExcelProperty(value = "知识库中检索的条数")
private Integer retrieveLimit;
/**
* 文本块大小
*/
@ExcelProperty(value = "文本块大小")
private Integer textBlockSize;
/**
* 向量库
*/
@ExcelProperty(value = "向量库")
private String vector;
/**
* 向量模型
*/
@ExcelProperty(value = "向量模型")
private String vectorModel;
}

2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java
View File

@@ -10,7 +10,7 @@ public interface EmbeddingService {
void removeByKid(String kid);
List<Double> getQueryVector(String query);
List<Double> getQueryVector(String query, String kid);
void createSchema(String kid);

6
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java
View File

@@ -55,4 +55,10 @@ public interface IKnowledgeInfoService {
* 删除知识库
*/
void removeKnowledge(String id);
/**
* 检查是否有删除权限
* @param knowledgeInfoList 知识列表
*/
void check(List<KnowledgeInfoVo> knowledgeInfoList);
}

6
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java
View File

@@ -24,7 +24,7 @@ public class EmbeddingServiceImpl implements EmbeddingService {
*/
@Override
public void storeEmbeddings(List<String> chunkList, String kid, String docId,List<String> fidList) {
List<List<Double>> vectorList = vectorization.batchVectorization(chunkList);
List<List<Double>> vectorList = vectorization.batchVectorization(chunkList, kid);
vectorStore.storeEmbeddings(chunkList,vectorList,kid,docId,fidList);
}
@@ -39,8 +39,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
}
@Override
public List<Double> getQueryVector(String query) {
List<Double> queryVector = vectorization.singleVectorization(query);
public List<Double> getQueryVector(String query, String kid) {
List<Double> queryVector = vectorization.singleVectorization(query,kid);
return queryVector;
}

18
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java
View File

@@ -4,16 +4,21 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import lombok.RequiredArgsConstructor;
import org.ruoyi.common.core.domain.model.LoginUser;
import org.ruoyi.common.core.utils.MapstructUtils;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.common.mybatis.core.page.PageQuery;
import org.ruoyi.common.mybatis.core.page.TableDataInfo;
import org.ruoyi.common.satoken.utils.LoginHelper;
import org.ruoyi.knowledge.domain.KnowledgeAttach;
import org.ruoyi.knowledge.domain.bo.KnowledgeAttachBo;
import org.ruoyi.knowledge.domain.vo.KnowledgeAttachVo;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.mapper.KnowledgeAttachMapper;
import org.ruoyi.knowledge.mapper.KnowledgeFragmentMapper;
import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper;
import org.ruoyi.knowledge.service.IKnowledgeAttachService;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.stereotype.Service;
import java.util.Collection;
@@ -35,6 +40,11 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
private final KnowledgeFragmentMapper fragmentMapper;
private final KnowledgeInfoMapper knowledgeInfoMapper;
private final IKnowledgeInfoService knowledgeInfoService;
/**
* 查询知识库附件
*/
@@ -117,8 +127,12 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
@Override
public void removeKnowledgeAttach(String kid) {
HashMap<String, Object> map = new HashMap<>();
map.put("kid", kid);
LoginUser loginUser = LoginHelper.getLoginUser();
Map<String,Object> map = new HashMap<>();
map.put("kid",kid);
List<KnowledgeInfoVo> knowledgeInfoList = knowledgeInfoMapper.selectVoByMap(map);
knowledgeInfoService.check(knowledgeInfoList);
baseMapper.deleteByMap(map);
fragmentMapper.deleteByMap(map);
}

48
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java
View File

@@ -1,16 +1,16 @@
package org.ruoyi.knowledge.service.impl;
import cn.hutool.core.util.RandomUtil;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.github.ollama4j.OllamaAPI;
import io.github.ollama4j.exceptions.OllamaBaseException;
import io.github.ollama4j.models.chat.OllamaChatMessageRole;
import io.github.ollama4j.models.chat.OllamaChatRequestBuilder;
import io.github.ollama4j.models.chat.OllamaChatRequestModel;
import io.github.ollama4j.models.chat.OllamaChatResult;
import lombok.RequiredArgsConstructor;
import org.ruoyi.common.core.domain.model.LoginUser;
import org.ruoyi.common.core.utils.MapstructUtils;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.common.mybatis.core.page.PageQuery;
@@ -21,8 +21,6 @@ import org.ruoyi.knowledge.chain.loader.ResourceLoaderFactory;
import org.ruoyi.knowledge.domain.KnowledgeAttach;
import org.ruoyi.knowledge.domain.KnowledgeFragment;
import org.ruoyi.knowledge.domain.KnowledgeInfo;
import org.ruoyi.knowledge.domain.bo.KnowledgeAttachBo;
import org.ruoyi.knowledge.domain.bo.KnowledgeFragmentBo;
import org.ruoyi.knowledge.domain.bo.KnowledgeInfoBo;
import org.ruoyi.knowledge.domain.req.KnowledgeInfoUploadRequest;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
@@ -30,14 +28,11 @@ import org.ruoyi.knowledge.mapper.KnowledgeAttachMapper;
import org.ruoyi.knowledge.mapper.KnowledgeFragmentMapper;
import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper;
import org.ruoyi.knowledge.service.EmbeddingService;
import org.ruoyi.knowledge.service.IKnowledgeAttachService;
import org.ruoyi.knowledge.service.IKnowledgeFragmentService;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.time.LocalDateTime;
import java.util.*;
/**
@@ -88,12 +83,13 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
}
private LambdaQueryWrapper<KnowledgeInfo> buildQueryWrapper(KnowledgeInfoBo bo) {
Map<String, Object> params = bo.getParams();
LambdaQueryWrapper<KnowledgeInfo> lqw = Wrappers.lambdaQuery();
lqw.eq(StringUtils.isNotBlank(bo.getKid()), KnowledgeInfo::getKid, bo.getKid());
lqw.eq(bo.getUid() != null, KnowledgeInfo::getUid, bo.getUid());
lqw.like(StringUtils.isNotBlank(bo.getKname()), KnowledgeInfo::getKname, bo.getKname());
lqw.eq(StringUtils.isNotBlank(bo.getDescription()), KnowledgeInfo::getDescription, bo.getDescription());
// 查询公开的知识库
lqw.or(wrapper -> wrapper.eq(KnowledgeInfo::getShare, "1"));
return lqw;
}
@@ -151,7 +147,7 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
List<String> fids = new ArrayList<>();
try {
content = resourceLoader.getContent(file.getInputStream());
chunkList = resourceLoader.getChunkList(content);
chunkList = resourceLoader.getChunkList(content, kid);
for (int i = 0; i < chunkList.size(); i++) {
String fid = RandomUtil.randomString(16);
fids.add(fid);
@@ -179,6 +175,8 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
Map<String,Object> map = new HashMap<>();
map.put("kid",id);
List<KnowledgeInfoVo> knowledgeInfoList = baseMapper.selectVoByMap(map);
check(knowledgeInfoList);
// 删除知识库
baseMapper.deleteByMap(map);
// 删除附件和知识片段
@@ -188,31 +186,13 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
embeddingService.removeByKid(id);
}
/**
* 将文本块转换为预训练数据
* @param chunk 解析文本块
*/
public String convertTextBlockToPretrainData(String chunk){
String host = "http://localhost:11434/";
OllamaAPI ollama = new OllamaAPI(host);
OllamaChatRequestBuilder builder = OllamaChatRequestBuilder.getInstance("qwen2.5:7b");
// 设置超时时间
ollama.setRequestTimeoutSeconds(100);
// create first user question
String json = "instruction:用户指令,根据语义提取一个关键词;input:用户输入,根据语义提取多个关键词;output:输出文本内容";
OllamaChatRequestModel requestModel = builder.withMessage
(OllamaChatMessageRole.USER, "文本："+chunk+"理解文本内容，并且将文本内容转换为:"+json+",输出JSON格式，不要包含其他无关内容,内部使用无需脱敏")
.build();
// start conversation with model
OllamaChatResult chatResult = null;
try {
chatResult = ollama.chat(requestModel);
} catch (Exception e) {
System.out.println("解析失败!");
@Override
public void check(List<KnowledgeInfoVo> knowledgeInfoList){
LoginUser loginUser = LoginHelper.getLoginUser();
for (KnowledgeInfoVo knowledgeInfoVo : knowledgeInfoList) {
if(!knowledgeInfoVo.getUid().equals(loginUser.getUserId())){
throw new SecurityException("权限不足");
}
return chatResult.getResponse();
}
}
}

4
ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java
View File

@@ -36,8 +36,8 @@ public class ChatConfigController extends BaseController {
*/
@GetMapping("/list")
@SaCheckPermission("system:config:list")
public List<ChatConfigVo> list(ChatConfigBo bo) {
return chatConfigService.queryList(bo);
public R<List<ChatConfigVo>> list(ChatConfigBo bo) {
return R.ok(chatConfigService.queryList(bo));
}
/**

7
ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java
View File

@@ -21,14 +21,9 @@ public class TranslationRequest {
*/
private String model;
/**
* 源语言
*/
private String sourceLanguage;
/**
* 目标语言
*/
private String targetLanguage;
private String targetLanguage;
}

16
ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java
View File

@@ -553,7 +553,21 @@ public class SseServiceImpl implements ISseService {
List<Message> messageList = new ArrayList<>();
Message sysMessage = Message.builder().role(Message.Role.SYSTEM).content("作为英汉翻译，您的任务是准确地在两种语言之间翻译文本。翻译时，请注意上下文，准确解释成语和谚语。如果连续收到多个英文单词，请默认将其翻译成中文句子。但如果前面有'phrase:’，则应翻译为短语;如果有'norma!:'，则翻译为多个无关的单词。您的翻译应接近母语者的水平，并考虑用户要求的特定语言风格或语气。避免使用冒犯性词汇，必要时用x替换。提供翻译时，请用中文解释每句话的时态、从句、主语、谓语、宾语、特殊短语和谚语对于需要翻译的短语或单词，请提供来源(词典)。如果要求翻译多个短语，请用|符号分隔。请记住:您是英汉翻译，不是汉汉翻译或英英翻译。提交前请仔细检查和修订答案,回复控制在50字以内").build();
Message sysMessage = Message.builder().role(Message.Role.SYSTEM).content("你是一名翻译老师\n" +
"\n" +
"请将用户输入词语翻译成{" + translationRequest.getTargetLanguage() + "}\n" +
"\n" +
"让我们一步一步来思考\n" +
"==示例输出==\n" +
"**翻译** : <这里显示翻译成英语的结果>\n" +
"\n" +
"**造句** : What's the weather like today? Use the 'Weather Query' plugin to find out instantly! <造一个英语句子>\n" +
"\n" +
"**同义词** : Add-on、Extension、Module <这里显示1-3个英文的同义词>\n" +
"\n" +
"==示例结束==\n" +
"\n" +
"注意：请严格按示例进行输出").build();
messageList.add(sysMessage);
Message message = Message.builder().role(Message.Role.USER).content(translationRequest.getPrompt()).build();
messageList.add(message);

1
script/sql/update/update20250302.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE `knowledge_info` ADD COLUMN `share` tinyint(4) NULL DEFAULT NULL COMMENT '是否公开知识库（0 否 1是）' AFTER `kname`;