diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 25b312ef..00000000 --- a/.editorconfig +++ /dev/null @@ -1,18 +0,0 @@ -# http://editorconfig.org -root = true - -# 空格替代Tab缩进在各种编辑工具下效果一致 -[*] -indent_style = space -indent_size = 4 -charset = utf-8 -end_of_line = lf -trim_trailing_whitespace = true -insert_final_newline = true - -[*.{json,yml,yaml}] -indent_size = 2 - -[*.md] -insert_final_newline = false -trim_trailing_whitespace = false diff --git a/ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java b/ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java index 76e59963..1768a697 100644 --- a/ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java +++ b/ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java @@ -81,7 +81,7 @@ public class KnowledgeController extends BaseController { List messages = chatRequest.getMessages(); String content = messages.get(messages.size() - 1).getContent().toString(); List nearestList; - List queryVector = embeddingService.getQueryVector(content); + List queryVector = embeddingService.getQueryVector(content, chatRequest.getKid()); nearestList = vectorStore.nearest(queryVector,chatRequest.getKid()); for (String prompt : nearestList) { Message sysMessage = Message.builder().content(prompt).role(Message.Role.USER).build(); diff --git a/ruoyi-admin/src/main/resources/application-dev.yml b/ruoyi-admin/src/main/resources/application-dev.yml index 434c6f40..6608a60f 100644 --- a/ruoyi-admin/src/main/resources/application-dev.yml +++ b/ruoyi-admin/src/main/resources/application-dev.yml @@ -27,7 +27,7 @@ spring: driverClassName: com.mysql.cj.jdbc.Driver url: jdbc:mysql://43.139.70.230:3306/ruoyi-ai?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true username: ruoyi-ai - password: ruoyi-ai + password: eCaZ278N62k6fhYj hikari: # 最大连接池数量 diff --git a/ruoyi-admin/src/main/resources/application.yml b/ruoyi-admin/src/main/resources/application.yml index bbaec244..26173564 100644 --- a/ruoyi-admin/src/main/resources/application.yml +++ b/ruoyi-admin/src/main/resources/application.yml @@ -310,11 +310,6 @@ wx: # 企业微信应用 wechat: - # 是否使用微信 true/false - enable: true - # 生成的登录二维码路径 默认与项目同级 - qrPath: "./" - # 企业微信应用 cp: corpId: appConfigs: @@ -323,28 +318,5 @@ wechat: token: '' aesKey: '' -# 知识库配置 -chain: - split: - chunk: - endspliter: "" - # 分块文本大小 - size: 200 - overlay: 30 - qaspliter: "###" - # 知识库中检索的条数 - limits: 5 - vector: - model: 'text-embedding-3-small' - store: - type: weaviate - weaviate: - protocol: http - host: 127.0.0.1:6038 - classname: LocalKnowledge - milvus: - host: 127.0.0.1 - port: 19530 - dimension: 1536 - collection: LocalKnowledge + diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java index 0ca7ea6e..5a74978d 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java @@ -31,7 +31,7 @@ public class CodeFileLoader implements ResourceLoader{ return stringBuffer.toString(); } @Override - public List getChunkList(String content){ - return textSplitter.split(content); + public List getChunkList(String content, String kid){ + return textSplitter.split(content, kid); } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java index 5bedf5b7..cec04fe2 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java @@ -10,7 +10,7 @@ public class CsvFileLoader implements ResourceLoader{ } @Override - public List getChunkList(String content) { + public List getChunkList(String content, String kid) { return null; } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java index 4682fee9..1bbbc7c8 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java @@ -10,7 +10,7 @@ public class FolderLoader implements ResourceLoader{ } @Override - public List getChunkList(String content) { + public List getChunkList(String content, String kid) { return null; } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java index ae86ac42..dd6b4496 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java @@ -10,7 +10,7 @@ public class GithubLoader implements ResourceLoader{ } @Override - public List getChunkList(String content) { + public List getChunkList(String content, String kid) { return null; } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java index 5cd1d6e0..5c69389c 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java @@ -10,7 +10,7 @@ public class JsonFileLoader implements ResourceLoader{ } @Override - public List getChunkList(String content) { + public List getChunkList(String content, String kid) { return null; } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java index cecdf052..24d43ff2 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java @@ -31,7 +31,7 @@ public class MarkDownFileLoader implements ResourceLoader{ return stringBuffer.toString(); } @Override - public List getChunkList(String content){ - return textSplitter.split(content); + public List getChunkList(String content, String kid){ + return textSplitter.split(content, kid); } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java index 83566de3..0acf355e 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java @@ -28,7 +28,7 @@ public class PdfFileLoader implements ResourceLoader{ } @Override - public List getChunkList(String content) { - return characterTextSplitter.split(content); + public List getChunkList(String content, String kid) { + return characterTextSplitter.split(content, kid); } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java index 32f41305..9ef4b11d 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java @@ -7,6 +7,8 @@ import java.util.List; * 资源载入 */ public interface ResourceLoader { + String getContent(InputStream inputStream); - List getChunkList(String content); + + List getChunkList(String content, String kid); } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java index a2e74fa0..95b8929a 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java @@ -31,7 +31,7 @@ public class TextFileLoader implements ResourceLoader{ return stringBuffer.toString(); } @Override - public List getChunkList(String content){ - return textSplitter.split(content); + public List getChunkList(String content, String kid){ + return textSplitter.split(content, kid); } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java index 7b9d9200..c31b9e6c 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java @@ -30,8 +30,8 @@ public class WordLoader implements ResourceLoader{ } @Override - public List getChunkList(String content) { - return textSplitter.split(content); + public List getChunkList(String content, String kid) { + return textSplitter.split(content, kid); } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/retrieve/PromptRetrieverProperties.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/retrieve/PromptRetrieverProperties.java deleted file mode 100644 index 3b62c6b6..00000000 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/retrieve/PromptRetrieverProperties.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.ruoyi.knowledge.chain.retrieve; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class PromptRetrieverProperties { - /** - * 从知识库中检索的条数,limits 应大于 num - */ - @Value("${chain.limits}") - private int limits; - -} diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java index 12db5139..cf86b34a 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java @@ -1,7 +1,10 @@ package org.ruoyi.knowledge.chain.split; -import lombok.AllArgsConstructor; +import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; +import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo; +import org.ruoyi.knowledge.service.IKnowledgeInfoService; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; @@ -10,38 +13,46 @@ import java.util.Arrays; import java.util.List; @Component -@AllArgsConstructor @Slf4j @Primary -public class CharacterTextSplitter implements TextSplitter{ - private final SplitterProperties splitterProperties; +public class CharacterTextSplitter implements TextSplitter { + + @Lazy + @Resource + private IKnowledgeInfoService knowledgeInfoService; + @Override - public List split(String content) { + public List split(String content, String kid) { + // 从知识库表中获取配置 + KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid)); + String knowledgeSeparator = knowledgeInfoVo.getKnowledgeSeparator(); + int textBlockSize = knowledgeInfoVo.getTextBlockSize(); + int overlapChar = knowledgeInfoVo.getOverlapChar(); List chunkList = new ArrayList<>(); - if (content.contains(splitterProperties.getEndspliter())){ + if (content.contains(knowledgeSeparator)) { // 按自定义分隔符切分 - String[] chunks = content.split(splitterProperties.getEndspliter()); + String[] chunks = content.split(knowledgeSeparator); chunkList.addAll(Arrays.asList(chunks)); - }else { + } else { int indexMin = 0; int len = content.length(); int i = 0; int right = 0; while (true) { - if (len > right ){ - int begin = i*splitterProperties.getSize() - splitterProperties.getOverlay(); - if (begin < indexMin){ + if (len > right) { + int begin = i * textBlockSize - overlapChar; + if (begin < indexMin) { begin = indexMin; } - int end = splitterProperties.getSize()*(i+1) + splitterProperties.getOverlay(); - if (end > len){ + int end = textBlockSize * (i + 1) + overlapChar; + if (end > len) { end = len; } - String chunk = content.substring(begin,end); + String chunk = content.substring(begin, end); chunkList.add(chunk); i++; - right = right + splitterProperties.getSize(); - }else { + right = right + textBlockSize; + } else { break; } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java index e68d61cc..209f5de6 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java @@ -11,7 +11,7 @@ import java.util.List; @Slf4j public class CodeTextSplitter implements TextSplitter{ @Override - public List split(String content) { + public List split(String content, String kid) { return null; } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java index 5c447813..89ea6536 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java @@ -11,7 +11,7 @@ import java.util.List; @Slf4j public class MarkdownTextSplitter implements TextSplitter{ @Override - public List split(String content) { + public List split(String content, String kid) { return null; } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/SplitterProperties.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/SplitterProperties.java deleted file mode 100644 index d0d9f5f0..00000000 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/SplitterProperties.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.ruoyi.knowledge.chain.split; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Data -@Component -@ConfigurationProperties(prefix = "chain.split.chunk") -public class SplitterProperties { - /** - * 分段标识符 - */ - private String endspliter; - - /** - * 提问分段标识符 - */ - private String qaspliter; - - /** - * 分块文本大小 - */ - private int size; - - /** - * 相邻块之间重叠的字符数(避免边界信息丢失) - */ - private int overlay; -} diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java index 5d3d71b5..86b3aafd 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java @@ -7,5 +7,12 @@ import java.util.List; */ public interface TextSplitter { - List split(String content); + /** + * 文本切分 + * + * @param content 文本内容 + * @param kid 知识库id + * @return 切分后的文本列表 + */ + List split(String content, String kid); } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java index dedc631a..696f2610 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java @@ -11,7 +11,7 @@ import java.util.List; @Slf4j public class TokenTextSplitter implements TextSplitter{ @Override - public List split(String content) { + public List split(String content, String kid) { return null; } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java index 2ac01435..0f2d0ba5 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java @@ -1,5 +1,6 @@ package org.ruoyi.knowledge.chain.vectorizer; +import jakarta.annotation.Resource; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -8,7 +9,10 @@ import org.ruoyi.common.chat.entity.embeddings.Embedding; import org.ruoyi.common.chat.entity.embeddings.EmbeddingResponse; import org.ruoyi.common.chat.openai.OpenAiStreamClient; +import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo; +import org.ruoyi.knowledge.service.IKnowledgeInfoService; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import java.math.BigDecimal; @@ -20,8 +24,9 @@ import java.util.List; @RequiredArgsConstructor public class OpenAiVectorization implements Vectorization { - @Value("${chain.vector.model}") - private String embeddingModel; + @Lazy + @Resource + private IKnowledgeInfoService knowledgeInfoService; @Getter private OpenAiStreamClient openAiStreamClient; @@ -29,12 +34,12 @@ public class OpenAiVectorization implements Vectorization { private final ChatConfig chatConfig; @Override - public List> batchVectorization(List chunkList) { + public List> batchVectorization(List chunkList, String kid) { openAiStreamClient = chatConfig.getOpenAiStreamClient(); - + KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid)); Embedding embedding = Embedding.builder() .input(chunkList) - .model(embeddingModel) + .model(knowledgeInfoVo.getVectorModel()) .build(); EmbeddingResponse embeddings = openAiStreamClient.embeddings(embedding); List> vectorList = new ArrayList<>(); @@ -50,10 +55,10 @@ public class OpenAiVectorization implements Vectorization { } @Override - public List singleVectorization(String chunk) { + public List singleVectorization(String chunk, String kid) { List chunkList = new ArrayList<>(); chunkList.add(chunk); - List> vectorList = batchVectorization(chunkList); + List> vectorList = batchVectorization(chunkList, kid); return vectorList.get(0); } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java index c770a16c..1c25ddeb 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java @@ -6,6 +6,7 @@ import java.util.List; * 向量化 */ public interface Vectorization { - List> batchVectorization(List chunkList); - List singleVectorization(String chunk); + List> batchVectorization(List chunkList, String kid); + + List singleVectorization(String chunk, String kid); } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java index 15221487..e41797fa 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java @@ -15,14 +15,14 @@ public class VectorizationWrapper implements Vectorization{ private final VectorizationFactory vectorizationFactory; @Override - public List> batchVectorization(List chunkList) { + public List> batchVectorization(List chunkList, String kid) { Vectorization embedding = vectorizationFactory.getEmbedding(); - return embedding.batchVectorization(chunkList); + return embedding.batchVectorization(chunkList, kid); } @Override - public List singleVectorization(String chunk) { + public List singleVectorization(String chunk, String kid) { Vectorization embedding = vectorizationFactory.getEmbedding(); - return embedding.singleVectorization(chunk); + return embedding.singleVectorization(chunk, kid); } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java index 74bef6fc..c083ef47 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java @@ -19,8 +19,10 @@ import io.milvus.param.partition.CreatePartitionParam; import io.milvus.response.QueryResultsWrapper; import io.milvus.response.SearchResultsWrapper; import jakarta.annotation.PostConstruct; +import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.ruoyi.common.core.service.ConfigService; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -32,26 +34,27 @@ import java.util.List; @Slf4j public class MilvusVectorStore implements VectorStore{ - - @Value("${chain.vector.store.milvus.host}") - private String milvusHost; - @Value("${chain.vector.store.milvus.port}") - private Integer milvausPort; - - @Value("${chain.vector.store.milvus.dimension}") - private Integer dimension; - - @Value("${chain.vector.store.milvus.collection}") - private String collectionName; - + private volatile Integer dimension; + private volatile String collectionName; private MilvusServiceClient milvusServiceClient; + @Resource + private ConfigService configService; + + @PostConstruct + public void loadConfig() { + this.dimension = Integer.parseInt(configService.getConfigValue("milvus", "dimension")); + this.collectionName = configService.getConfigValue("milvus", "collection"); + } + @PostConstruct public void init(){ + String milvusHost = configService.getConfigValue("milvus", "host"); + String milvausPort = configService.getConfigValue("milvus", "port"); milvusServiceClient = new MilvusServiceClient( ConnectParam.newBuilder() .withHost(milvusHost) - .withPort(milvausPort) + .withPort(Integer.parseInt(milvausPort)) .withDatabaseName("default") .build() ); diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java index 34782296..ae6e73c8 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java @@ -1,29 +1,37 @@ package org.ruoyi.knowledge.chain.vectorstore; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; +import org.ruoyi.knowledge.domain.KnowledgeInfo; +import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo; +import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper; import org.springframework.stereotype.Component; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + @Component @Slf4j +@RequiredArgsConstructor public class VectorStoreFactory { - @Value("${chain.vector.store.type}") - private String type; - private final WeaviateVectorStore weaviateVectorStore; private final MilvusVectorStore milvusVectorStore; - public VectorStoreFactory(WeaviateVectorStore weaviateVectorStore, MilvusVectorStore milvusVectorStore) { - this.weaviateVectorStore = weaviateVectorStore; - this.milvusVectorStore = milvusVectorStore; - } + private final KnowledgeInfoMapper knowledgeInfoMapper; - public VectorStore getVectorStore(){ - if ("weaviate".equals(type)){ + public VectorStore getVectorStore(String kid){ + KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoMapper.selectVoOne( + new LambdaQueryWrapper().eq(KnowledgeInfo::getKid,kid) + ); + String vectorModel = knowledgeInfoVo.getVector(); + if ("weaviate".equals(vectorModel)){ return weaviateVectorStore; - }else if ("milvus".equals(type)){ + }else if ("milvus".equals(vectorModel)){ return milvusVectorStore; } return null; diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java index 2c0a7325..eb6b1f4d 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java @@ -16,43 +16,43 @@ public class VectorStoreWrapper implements VectorStore{ private final VectorStoreFactory vectorStoreFactory; @Override public void storeEmbeddings(List chunkList, List> vectorList, String kid, String docId, List fidList) { - VectorStore vectorStore = vectorStoreFactory.getVectorStore(); + VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid); vectorStore.storeEmbeddings(chunkList, vectorList, kid, docId, fidList); } @Override public void removeByDocId(String kid, String docId) { - VectorStore vectorStore = vectorStoreFactory.getVectorStore(); + VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid); vectorStore.removeByDocId(kid,docId); } @Override public void removeByKid(String kid) { - VectorStore vectorStore = vectorStoreFactory.getVectorStore(); + VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid); vectorStore.removeByKid(kid); } @Override public List nearest(List queryVector, String kid) { - VectorStore vectorStore = vectorStoreFactory.getVectorStore(); + VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid); return vectorStore.nearest(queryVector,kid); } @Override public List nearest(String query, String kid) { - VectorStore vectorStore = vectorStoreFactory.getVectorStore(); + VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid); return vectorStore.nearest(query, kid); } @Override public void newSchema(String kid) { - VectorStore vectorStore = vectorStoreFactory.getVectorStore(); + VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid); vectorStore.newSchema(kid); } @Override public void removeByKidAndFid(String kid, String fid) { - VectorStore vectorStore = vectorStoreFactory.getVectorStore(); + VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid); vectorStore.removeByKidAndFid(kid, fid); } } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java index 335e4edb..8895a251 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java @@ -22,10 +22,15 @@ import io.weaviate.client.v1.schema.model.DataType; import io.weaviate.client.v1.schema.model.Property; import io.weaviate.client.v1.schema.model.Schema; import io.weaviate.client.v1.schema.model.WeaviateClass; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.ruoyi.knowledge.chain.retrieve.PromptRetrieverProperties; +import org.ruoyi.common.core.service.ConfigService; +import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo; +import org.ruoyi.knowledge.service.IKnowledgeInfoService; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -37,18 +42,23 @@ import java.util.Map; @Slf4j public class WeaviateVectorStore implements VectorStore{ - @Value("${chain.vector.store.weaviate.protocol}") - private String protocol; - @Value("${chain.vector.store.weaviate.host}") - private String host; + private volatile String protocol; + private volatile String host; + private volatile String className; - @Value("${chain.vector.store.weaviate.classname}") - private String className; + @Lazy + @Resource + private IKnowledgeInfoService knowledgeInfoService; - private final PromptRetrieverProperties promptRetrieverProperties; + @Lazy + @Resource + private ConfigService configService; - public WeaviateVectorStore(PromptRetrieverProperties promptRetrieverProperties) { - this.promptRetrieverProperties = promptRetrieverProperties; + @PostConstruct + public void loadConfig() { + this.protocol = configService.getConfigValue("weaviate", "protocol"); + this.host = configService.getConfigValue("weaviate", "host"); + this.className = configService.getConfigValue("weaviate", "classname"); } public WeaviateClient getClient(){ @@ -309,11 +319,12 @@ public class WeaviateVectorStore implements VectorStore{ .vector(vf) .distance(1.6f) // certainty = 1f - distance /2f .build(); + KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid)); Result result = client.graphQL().get() .withClassName(className + kid) .withFields(contentField,_additional) .withNearVector(nearVector) - .withLimit(promptRetrieverProperties.getLimits()) + .withLimit(knowledgeInfoVo.getRetrieveLimit()) .run(); LinkedTreeMap t = (LinkedTreeMap) result.getResult().getData(); LinkedTreeMap> l = (LinkedTreeMap>) t.get("Get"); @@ -342,12 +353,12 @@ public class WeaviateVectorStore implements VectorStore{ .concepts(new String[]{ query }) .distance(1.6f) // certainty = 1f - distance /2f .build(); - + KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid)); Result result = client.graphQL().get() .withClassName(className + kid) .withFields(contentField,_additional) .withNearText(nearText) - .withLimit(promptRetrieverProperties.getLimits()) + .withLimit(knowledgeInfoVo.getRetrieveLimit()) .run(); LinkedTreeMap t = (LinkedTreeMap) result.getResult().getData(); LinkedTreeMap> l = (LinkedTreeMap>) t.get("Get"); diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java index 0d9b51e3..fb8407df 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java @@ -2,6 +2,7 @@ package org.ruoyi.knowledge.domain; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import jakarta.validation.constraints.NotBlank; import lombok.Data; import java.io.Serial; @@ -43,6 +44,11 @@ public class KnowledgeInfo implements Serializable { */ private String kname; + /** + * 知识库名称 + */ + private String share; + /** * 描述 */ @@ -59,5 +65,38 @@ public class KnowledgeInfo implements Serializable { */ private Date createTime; + /** + * 知识分隔符 + */ + private String knowledgeSeparator; + /** + * 提问分隔符 + */ + private String questionSeparator; + + /** + * 重叠字符数 + */ + private Integer overlapChar; + + /** + * 知识库中检索的条数 + */ + private Integer retrieveLimit; + + /** + * 文本块大小 + */ + private Integer textBlockSize; + + /** + * 向量库 + */ + private String vector; + + /** + * 向量模型 + */ + private String vectorModel; } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java index bea7b25b..b606d823 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java @@ -43,11 +43,57 @@ public class KnowledgeInfoBo extends BaseEntity { @NotBlank(message = "知识库名称不能为空") private String kname; + /** + * 知识库名称 + */ + @NotBlank(message = "是否公开知识库") + private String share; + /** * 描述 */ @NotBlank(message = "描述不能为空") private String description; + /** + * 知识分隔符 + */ + @NotBlank(message = "知识分隔符不能为空") + private String knowledgeSeparator; + /** + * 提问分隔符 + */ + @NotBlank(message = "提问分隔符不能为空") + private String questionSeparator; + + /** + * 重叠字符数 + */ + @NotNull(message = "重叠字符数不能为空") + private Integer overlapChar; + + /** + * 知识库中检索的条数 + */ + @NotNull(message = "知识库中检索的条数不能为空") + private Integer retrieveLimit; + + /** + * 文本块大小 + */ + @NotNull(message = "文本块大小不能为空") + private Integer textBlockSize; + + /** + * 向量库 + */ + @NotBlank(message = "向量库不能为空") + private String vector; + + /** + * 向量模型 + */ + @NotBlank(message = "向量模型不能为空") + private String vectorModel; } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java index c541a823..a47b5d60 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java @@ -47,11 +47,56 @@ public class KnowledgeInfoVo implements Serializable { @ExcelProperty(value = "知识库名称") private String kname; + /** + * 知识库名称 + */ + private String share; + /** * 描述 */ @ExcelProperty(value = "描述") private String description; + /** + * 知识分隔符 + */ + @ExcelProperty(value = "知识分隔符") + private String knowledgeSeparator; + /** + * 提问分隔符 + */ + @ExcelProperty(value = "提问分隔符") + private String questionSeparator; + + /** + * 重叠字符数 + */ + @ExcelProperty(value = "重叠字符数") + private Integer overlapChar; + + /** + * 知识库中检索的条数 + */ + @ExcelProperty(value = "知识库中检索的条数") + private Integer retrieveLimit; + + /** + * 文本块大小 + */ + @ExcelProperty(value = "文本块大小") + private Integer textBlockSize; + + /** + * 向量库 + */ + @ExcelProperty(value = "向量库") + private String vector; + + /** + * 向量模型 + */ + @ExcelProperty(value = "向量模型") + private String vectorModel; } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java index 739c7e6a..a4dd30d5 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java @@ -10,7 +10,7 @@ public interface EmbeddingService { void removeByKid(String kid); - List getQueryVector(String query); + List getQueryVector(String query, String kid); void createSchema(String kid); diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java index 0c1b0909..121f8fb2 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java @@ -55,4 +55,10 @@ public interface IKnowledgeInfoService { * 删除知识库 */ void removeKnowledge(String id); + + /** + * 检查是否有删除权限 + * @param knowledgeInfoList 知识列表 + */ + void check(List knowledgeInfoList); } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java index 8850311f..73503a45 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java @@ -24,7 +24,7 @@ public class EmbeddingServiceImpl implements EmbeddingService { */ @Override public void storeEmbeddings(List chunkList, String kid, String docId,List fidList) { - List> vectorList = vectorization.batchVectorization(chunkList); + List> vectorList = vectorization.batchVectorization(chunkList, kid); vectorStore.storeEmbeddings(chunkList,vectorList,kid,docId,fidList); } @@ -39,8 +39,8 @@ public class EmbeddingServiceImpl implements EmbeddingService { } @Override - public List getQueryVector(String query) { - List queryVector = vectorization.singleVectorization(query); + public List getQueryVector(String query, String kid) { + List queryVector = vectorization.singleVectorization(query,kid); return queryVector; } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java index 07812d48..9bea5dd4 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java @@ -4,16 +4,21 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import lombok.RequiredArgsConstructor; +import org.ruoyi.common.core.domain.model.LoginUser; import org.ruoyi.common.core.utils.MapstructUtils; import org.ruoyi.common.core.utils.StringUtils; import org.ruoyi.common.mybatis.core.page.PageQuery; import org.ruoyi.common.mybatis.core.page.TableDataInfo; +import org.ruoyi.common.satoken.utils.LoginHelper; import org.ruoyi.knowledge.domain.KnowledgeAttach; import org.ruoyi.knowledge.domain.bo.KnowledgeAttachBo; import org.ruoyi.knowledge.domain.vo.KnowledgeAttachVo; +import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo; import org.ruoyi.knowledge.mapper.KnowledgeAttachMapper; import org.ruoyi.knowledge.mapper.KnowledgeFragmentMapper; +import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper; import org.ruoyi.knowledge.service.IKnowledgeAttachService; +import org.ruoyi.knowledge.service.IKnowledgeInfoService; import org.springframework.stereotype.Service; import java.util.Collection; @@ -35,6 +40,11 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService { private final KnowledgeFragmentMapper fragmentMapper; + private final KnowledgeInfoMapper knowledgeInfoMapper; + + private final IKnowledgeInfoService knowledgeInfoService; + + /** * 查询知识库附件 */ @@ -117,8 +127,12 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService { @Override public void removeKnowledgeAttach(String kid) { - HashMap map = new HashMap<>(); - map.put("kid", kid); + LoginUser loginUser = LoginHelper.getLoginUser(); + Map map = new HashMap<>(); + map.put("kid",kid); + List knowledgeInfoList = knowledgeInfoMapper.selectVoByMap(map); + knowledgeInfoService.check(knowledgeInfoList); + baseMapper.deleteByMap(map); fragmentMapper.deleteByMap(map); } diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java index 0f2b8421..3bd9390d 100644 --- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java +++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java @@ -1,16 +1,16 @@ package org.ruoyi.knowledge.service.impl; import cn.hutool.core.util.RandomUtil; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import io.github.ollama4j.OllamaAPI; -import io.github.ollama4j.exceptions.OllamaBaseException; import io.github.ollama4j.models.chat.OllamaChatMessageRole; import io.github.ollama4j.models.chat.OllamaChatRequestBuilder; import io.github.ollama4j.models.chat.OllamaChatRequestModel; import io.github.ollama4j.models.chat.OllamaChatResult; import lombok.RequiredArgsConstructor; +import org.ruoyi.common.core.domain.model.LoginUser; import org.ruoyi.common.core.utils.MapstructUtils; import org.ruoyi.common.core.utils.StringUtils; import org.ruoyi.common.mybatis.core.page.PageQuery; @@ -21,8 +21,6 @@ import org.ruoyi.knowledge.chain.loader.ResourceLoaderFactory; import org.ruoyi.knowledge.domain.KnowledgeAttach; import org.ruoyi.knowledge.domain.KnowledgeFragment; import org.ruoyi.knowledge.domain.KnowledgeInfo; -import org.ruoyi.knowledge.domain.bo.KnowledgeAttachBo; -import org.ruoyi.knowledge.domain.bo.KnowledgeFragmentBo; import org.ruoyi.knowledge.domain.bo.KnowledgeInfoBo; import org.ruoyi.knowledge.domain.req.KnowledgeInfoUploadRequest; import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo; @@ -30,14 +28,11 @@ import org.ruoyi.knowledge.mapper.KnowledgeAttachMapper; import org.ruoyi.knowledge.mapper.KnowledgeFragmentMapper; import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper; import org.ruoyi.knowledge.service.EmbeddingService; -import org.ruoyi.knowledge.service.IKnowledgeAttachService; -import org.ruoyi.knowledge.service.IKnowledgeFragmentService; import org.ruoyi.knowledge.service.IKnowledgeInfoService; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; -import java.time.LocalDateTime; import java.util.*; /** @@ -88,12 +83,13 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { } private LambdaQueryWrapper buildQueryWrapper(KnowledgeInfoBo bo) { - Map params = bo.getParams(); LambdaQueryWrapper lqw = Wrappers.lambdaQuery(); lqw.eq(StringUtils.isNotBlank(bo.getKid()), KnowledgeInfo::getKid, bo.getKid()); lqw.eq(bo.getUid() != null, KnowledgeInfo::getUid, bo.getUid()); lqw.like(StringUtils.isNotBlank(bo.getKname()), KnowledgeInfo::getKname, bo.getKname()); lqw.eq(StringUtils.isNotBlank(bo.getDescription()), KnowledgeInfo::getDescription, bo.getDescription()); + // 查询公开的知识库 + lqw.or(wrapper -> wrapper.eq(KnowledgeInfo::getShare, "1")); return lqw; } @@ -151,7 +147,7 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { List fids = new ArrayList<>(); try { content = resourceLoader.getContent(file.getInputStream()); - chunkList = resourceLoader.getChunkList(content); + chunkList = resourceLoader.getChunkList(content, kid); for (int i = 0; i < chunkList.size(); i++) { String fid = RandomUtil.randomString(16); fids.add(fid); @@ -179,6 +175,8 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { Map map = new HashMap<>(); map.put("kid",id); + List knowledgeInfoList = baseMapper.selectVoByMap(map); + check(knowledgeInfoList); // 删除知识库 baseMapper.deleteByMap(map); // 删除附件和知识片段 @@ -188,31 +186,13 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { embeddingService.removeByKid(id); } - /** - * 将文本块转换为预训练数据 - * @param chunk 解析文本块 - */ - public String convertTextBlockToPretrainData(String chunk){ - String host = "http://localhost:11434/"; - OllamaAPI ollama = new OllamaAPI(host); - OllamaChatRequestBuilder builder = OllamaChatRequestBuilder.getInstance("qwen2.5:7b"); - // 设置超时时间 - ollama.setRequestTimeoutSeconds(100); - // create first user question - String json = "instruction:用户指令,根据语义提取一个关键词;input:用户输入,根据语义提取多个关键词;output:输出文本内容"; - - OllamaChatRequestModel requestModel = builder.withMessage - (OllamaChatMessageRole.USER, "文本:"+chunk+"理解文本内容,并且将文本内容转换为:"+json+",输出JSON格式,不要包含其他无关内容,内部使用无需脱敏") - .build(); - - // start conversation with model - OllamaChatResult chatResult = null; - try { - chatResult = ollama.chat(requestModel); - } catch (Exception e) { - System.out.println("解析失败!"); + @Override + public void check(List knowledgeInfoList){ + LoginUser loginUser = LoginHelper.getLoginUser(); + for (KnowledgeInfoVo knowledgeInfoVo : knowledgeInfoList) { + if(!knowledgeInfoVo.getUid().equals(loginUser.getUserId())){ + throw new SecurityException("权限不足"); } - return chatResult.getResponse(); + } } - } diff --git a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java index 4af23486..a77a0d3e 100644 --- a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java +++ b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java @@ -36,8 +36,8 @@ public class ChatConfigController extends BaseController { */ @GetMapping("/list") @SaCheckPermission("system:config:list") - public List list(ChatConfigBo bo) { - return chatConfigService.queryList(bo); + public R> list(ChatConfigBo bo) { + return R.ok(chatConfigService.queryList(bo)); } /** diff --git a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java index 5148f34e..04fa785a 100644 --- a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java +++ b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java @@ -21,14 +21,9 @@ public class TranslationRequest { */ private String model; - /** - * 源语言 - */ - private String sourceLanguage; - /** * 目标语言 */ - private String targetLanguage; + private String targetLanguage; } diff --git a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java index 7ada900f..e2f02b58 100644 --- a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java +++ b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java @@ -553,7 +553,21 @@ public class SseServiceImpl implements ISseService { List messageList = new ArrayList<>(); - Message sysMessage = Message.builder().role(Message.Role.SYSTEM).content("作为英汉翻译,您的任务是准确地在两种语言之间翻译文本。翻译时,请注意上下文,准确解释成语和谚语。如果连续收到多个英文单词,请默认将其翻译成中文句子。但如果前面有'phrase:’,则应翻译为短语;如果有'norma!:',则翻译为多个无关的单词。您的翻译应接近母语者的水平,并考虑用户要求的特定语言风格或语气。避免使用冒犯性词汇,必要时用x替换。提供翻译时,请用中文解释每句话的时态、从句、主语、谓语、宾语、特殊短语和谚语对于需要翻译的短语或单词,请提供来源(词典)。如果要求翻译多个短语,请用|符号分隔。请记住:您是英汉翻译,不是汉汉翻译或英英翻译。提交前请仔细检查和修订答案,回复控制在50字以内").build(); + Message sysMessage = Message.builder().role(Message.Role.SYSTEM).content("你是一名翻译老师\n" + + "\n" + + "请将用户输入词语翻译成{" + translationRequest.getTargetLanguage() + "}\n" + + "\n" + + "让我们一步一步来思考\n" + + "==示例输出==\n" + + "**翻译** : <这里显示翻译成英语的结果>\n" + + "\n" + + "**造句** : What's the weather like today? Use the 'Weather Query' plugin to find out instantly! <造一个英语句子>\n" + + "\n" + + "**同义词** : Add-on、Extension、Module <这里显示1-3个英文的同义词>\n" + + "\n" + + "==示例结束==\n" + + "\n" + + "注意:请严格按示例进行输出").build(); messageList.add(sysMessage); Message message = Message.builder().role(Message.Role.USER).content(translationRequest.getPrompt()).build(); messageList.add(message); diff --git a/script/sql/update/update20250302.sql b/script/sql/update/update20250302.sql new file mode 100644 index 00000000..b3aa42b8 --- /dev/null +++ b/script/sql/update/update20250302.sql @@ -0,0 +1 @@ +ALTER TABLE `knowledge_info` ADD COLUMN `share` tinyint(4) NULL DEFAULT NULL COMMENT '是否公开知识库(0 否 1是)' AFTER `kname`;