xieyinlu/ruoyi-ai
1
0
Fork 0
mirror of https://gitcode.com/ageerle/ruoyi-ai.git synced 2026-09-13 00:14:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(rag): 统一文档切割配置并支持重新解析

Browse Source
This commit is contained in:
evo
2026-07-21 12:18:19 +08:00
parent 177375a7b7
commit cf5423f69f
26 changed files with 272 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/knowledge/KnowledgeAttachController.java
View File

@@ -9,6 +9,7 @@ import cn.dev33.satoken.annotation.SaCheckPermission;
import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo;
import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo;
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
import org.ruoyi.service.knowledge.IKnowledgeAttachService;
import org.springframework.web.bind.annotation.*;
import org.springframework.validation.annotation.Validated;
@@ -129,4 +130,12 @@ public class KnowledgeAttachController extends BaseController {
knowledgeAttachService.parse(id);
return R.ok();
}
@SaCheckPermission("system:attach:edit")
@Log(title = "知识库附件批量重新解析", businessType = BusinessType.UPDATE)
@PostMapping("/reparse/knowledge/{knowledgeId}")
@RepeatSubmit()
public R<KnowledgeReparseVo> reparseKnowledge(@PathVariable Long knowledgeId) {
return R.ok(knowledgeAttachService.reparseKnowledge(knowledgeId));
}
}

4
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/knowledge/KnowledgeReparseVo.java Normal file
View File

@@ -0,0 +1,4 @@
package org.ruoyi.domain.vo.knowledge;
public record KnowledgeReparseVo(int submitted, int skipped, int total) {
}

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/ResourceLoaderFactory.java
View File

@@ -17,7 +17,7 @@ public class ResourceLoaderFactory {
private final ExcelTextSplitter excelTextSplitter;
public ResourceLoader getLoaderByFileType(String fileType) {
fileType = StringUtils.removeStart(fileType, ".");
fileType = StringUtils.lowerCase(StringUtils.removeStart(StringUtils.trim(fileType), "."));
if (FileTypeConstants.isTextFile(fileType)) {
return new TextFileLoader(characterTextSplitter);
} else if (FileTypeConstants.isWord(fileType)) {

20
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/DocumentSplitConfig.java Normal file
View File

@@ -0,0 +1,20 @@
package org.ruoyi.service.knowledge;
import org.ruoyi.common.core.exception.ServiceException;
/** Immutable snapshot of the split settings used for one parse operation. */
public record DocumentSplitConfig(String separator, int blockSize, int overlap, String fileType) {
public static final int DEFAULT_BLOCK_SIZE = 1000;
public static final int DEFAULT_OVERLAP = 50;
public DocumentSplitConfig {
if (blockSize <= 0) {
throw new ServiceException("文本块大小必须大于0");
}
if (overlap < 0 || overlap >= blockSize) {
throw new ServiceException("重叠字符数必须大于等于0且小于文本块大小");
}
fileType = fileType == null ? "" : fileType.strip().replaceFirst("^\\.", "").toLowerCase();
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/IKnowledgeAttachService.java
View File

@@ -5,6 +5,7 @@ import org.ruoyi.common.mybatis.core.page.PageQuery;
import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo;
import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo;
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
import java.util.Collection;
import java.util.List;
@@ -79,4 +80,6 @@ public interface IKnowledgeAttachService {
* @param id 附件ID
*/
void parse(Long id);
KnowledgeReparseVo reparseKnowledge(Long knowledgeId);
}

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/ResourceLoader.java
View File

@@ -10,5 +10,5 @@ public interface ResourceLoader {
String getContent(InputStream inputStream);
List<String> getChunkList(String content, String kid);
List<String> getChunkList(String content, DocumentSplitConfig config);
}

4
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/TextSplitter.java
View File

@@ -11,8 +11,8 @@ public interface TextSplitter {
* 文本切分
*
* @param content 文本内容
* @param kid 知识库id
* @param config 本次解析的分片配置快照
* @return 切分后的文本列表
*/
List<String> split(String content, String kid);
List<String> split(String content, DocumentSplitConfig config);
}

42
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java
View File

@@ -27,12 +27,14 @@ import org.ruoyi.domain.entity.knowledge.KnowledgeFragment;
import org.ruoyi.domain.vo.knowledge.DocFragmentCountVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
import org.ruoyi.factory.ResourceLoaderFactory;
import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper;
import org.ruoyi.mapper.knowledge.KnowledgeFragmentMapper;
import org.ruoyi.service.knowledge.IKnowledgeAttachService;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.vector.VectorStoreService;
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
import org.springframework.scheduling.annotation.Async;
@@ -184,10 +186,17 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
@Override
public void parse(Long id) {
KnowledgeAttach attach = baseMapper.selectById(id);
if (attach == null || (!KnowledgeAttachStatus.WAITING.getCode().equals(attach.getStatus()) && !KnowledgeAttachStatus.FAILED.getCode().equals(attach.getStatus()))) {
if (attach == null || KnowledgeAttachStatus.PARSING.getCode().equals(attach.getStatus())) {
return;
}
int claimed = baseMapper.update(null, Wrappers.<KnowledgeAttach>lambdaUpdate()
.set(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode())
.set(KnowledgeAttach::getRemark, null)
.eq(KnowledgeAttach::getId, id)
.ne(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode()));
if (claimed == 0) return;
try {
attach.setStatus(KnowledgeAttachStatus.PARSING.getCode()); // 解析中
baseMapper.updateById(attach);
@@ -196,6 +205,16 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
Long knowledgeId = attach.getKnowledgeId();
String docId = attach.getDocId();
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
if (knowledgeInfoVo == null) {
throw new ServiceException("知识库不存在: " + knowledgeId);
}
int blockSize = knowledgeInfoVo.getTextBlockSize() == null
? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : knowledgeInfoVo.getTextBlockSize().intValue();
int overlap = knowledgeInfoVo.getOverlapChar() == null
? DocumentSplitConfig.DEFAULT_OVERLAP : knowledgeInfoVo.getOverlapChar().intValue();
DocumentSplitConfig splitConfig = new DocumentSplitConfig(
knowledgeInfoVo.getSeparator(), blockSize, overlap, attach.getType());
List<KnowledgeFragment> oldFragments = knowledgeFragmentMapper.selectList(
Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
@@ -210,7 +229,7 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
try (InputStream inputStream = new URL(ossDTO.getUrl()).openStream()) {
content = resourceLoader.getContent(inputStream);
}
List<String> chunkList = resourceLoader.getChunkList(content, String.valueOf(knowledgeId));
List<String> chunkList = resourceLoader.getChunkList(content, splitConfig);
if (CollUtil.isEmpty(chunkList)) {
throw new RuntimeException("文档分片结果为空，请检查文档内容或分片器是否支持该文件类型");
@@ -231,7 +250,6 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
knowledgeFragment.setCreateTime(new Date());
knowledgeFragmentList.add(knowledgeFragment);
}
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel());
StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo();
@@ -275,4 +293,22 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
baseMapper.updateById(attach);
}
}
@Override
public KnowledgeReparseVo reparseKnowledge(Long knowledgeId) {
List<KnowledgeAttach> attachments = baseMapper.selectList(
Wrappers.<KnowledgeAttach>lambdaQuery().eq(KnowledgeAttach::getKnowledgeId, knowledgeId));
int submitted = 0;
int skipped = 0;
IKnowledgeAttachService proxy = SpringUtils.getBean(IKnowledgeAttachService.class);
for (KnowledgeAttach attachment : attachments) {
if (KnowledgeAttachStatus.PARSING.getCode().equals(attachment.getStatus())) {
skipped++;
} else {
proxy.parse(attachment.getId());
submitted++;
}
}
return new KnowledgeReparseVo(submitted, skipped, attachments.size());
}
}

6
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeInfoServiceImpl.java
View File

@@ -18,6 +18,7 @@ import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.common.core.service.OssService;
import java.util.List;
@@ -152,6 +153,11 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
* 保存前的数据校验
*/
private void validEntityBeforeSave(KnowledgeInfo entity){
int blockSize = entity.getTextBlockSize() == null
? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : entity.getTextBlockSize().intValue();
int overlap = entity.getOverlapChar() == null
? DocumentSplitConfig.DEFAULT_OVERLAP : entity.getOverlapChar().intValue();
new DocumentSplitConfig(entity.getSeparator(), blockSize, overlap, "");
//TODO 做一些数据校验,如唯一约束
}

11
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CodeFileLoader.java
View File

@@ -3,13 +3,16 @@ package org.ruoyi.service.knowledge.impl.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.ruoyi.common.core.exception.ServiceException;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.List;
@Component
@@ -21,20 +24,20 @@ public class CodeFileLoader implements ResourceLoader {
@Override
public String getContent(InputStream inputStream) {
StringBuffer stringBuffer = new StringBuffer();
try (InputStreamReader reader = new InputStreamReader(inputStream);
try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
BufferedReader bufferedReader = new BufferedReader(reader)) {
String line;
while ((line = bufferedReader.readLine()) != null) {
stringBuffer.append(line).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
throw new ServiceException("读取代码文件失败", e);
}
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CsvFileLoader.java
View File

@@ -1,6 +1,7 @@
package org.ruoyi.service.knowledge.impl.loader;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.io.InputStream;
import java.util.List;
@@ -12,7 +13,7 @@ public class CsvFileLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return null;
}
}

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/ExcelFileLoader.java
View File

@@ -6,6 +6,7 @@ import dev.langchain4j.data.document.parser.apache.tika.ApacheTikaDocumentParser
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
@@ -38,7 +39,7 @@ public class ExcelFileLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/FolderLoader.java
View File

@@ -1,6 +1,7 @@
package org.ruoyi.service.knowledge.impl.loader;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.io.InputStream;
import java.util.List;
@@ -12,7 +13,7 @@ public class FolderLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return null;
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/GithubLoader.java
View File

@@ -1,6 +1,7 @@
package org.ruoyi.service.knowledge.impl.loader;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.io.InputStream;
import java.util.List;
@@ -12,7 +13,7 @@ public class GithubLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return null;
}
}

3
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/JsonFileLoader.java
View File

@@ -1,6 +1,7 @@
package org.ruoyi.service.knowledge.impl.loader;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.io.InputStream;
import java.util.List;
@@ -12,7 +13,7 @@ public class JsonFileLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return null;
}
}

11
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/MarkDownFileLoader.java
View File

@@ -3,13 +3,16 @@ package org.ruoyi.service.knowledge.impl.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.ruoyi.common.core.exception.ServiceException;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.List;
@Component
@@ -21,20 +24,20 @@ public class MarkDownFileLoader implements ResourceLoader {
@Override
public String getContent(InputStream inputStream) {
StringBuffer stringBuffer = new StringBuffer();
try (InputStreamReader reader = new InputStreamReader(inputStream);
try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
BufferedReader bufferedReader = new BufferedReader(reader)) {
String line;
while ((line = bufferedReader.readLine()) != null) {
stringBuffer.append(line).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
throw new ServiceException("读取 Markdown 文件失败", e);
}
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/PdfFileLoader.java
View File

@@ -6,6 +6,7 @@ import org.apache.pdfbox.io.RandomAccessReadBuffer;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
@@ -32,7 +33,7 @@ public class PdfFileLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
return characterTextSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return characterTextSplitter.split(content, config);
}
}

12
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/TextFileLoader.java
View File

@@ -3,7 +3,9 @@ package org.ruoyi.service.knowledge.impl.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.ruoyi.common.core.exception.ServiceException;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
@@ -22,18 +24,16 @@ public class TextFileLoader implements ResourceLoader {
@Override
public String getContent(InputStream inputStream) {
String stringBuffer = "";
try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
BufferedReader bufferedReader = new BufferedReader(reader)) {
stringBuffer = bufferedReader.lines().collect(Collectors.joining());
return bufferedReader.lines().collect(Collectors.joining("\n"));
} catch (IOException e) {
e.printStackTrace();
throw new ServiceException("读取文本文件失败", e);
}
return stringBuffer;
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/WordLoader.java
View File

@@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.ruoyi.service.knowledge.ResourceLoader;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
@@ -32,8 +33,8 @@ public class WordLoader implements ResourceLoader {
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
public List<String> getChunkList(String content, DocumentSplitConfig config) {
return textSplitter.split(content, config);
}
}

75
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java
View File

@@ -1,86 +1,17 @@
package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Component
@Slf4j
@Primary
@AllArgsConstructor
public class CharacterTextSplitter implements TextSplitter {
private final IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content, String kid) {
// 默认配置值
String knowledgeSeparator = "#";
int textBlockSize = 1000;
int overlapChar = 50;
// 根据知识库ID查询配置，覆盖默认值
if (StringUtils.isNotBlank(kid)) {
try {
KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
if (info != null) {
if (StringUtils.isNotBlank(info.getSeparator())) {
knowledgeSeparator = info.getSeparator();
}
if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
textBlockSize = info.getTextBlockSize().intValue();
}
if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
overlapChar = info.getOverlapChar().intValue();
}
}
} catch (Exception e) {
log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
}
}
List<String> chunkList = new ArrayList<>();
if (StringUtils.isNotBlank(knowledgeSeparator) && content.contains(knowledgeSeparator)) {
// 按自定义分隔符切分（字面量匹配，避免分隔符被当作正则）
String[] chunks = content.split(java.util.regex.Pattern.quote(knowledgeSeparator));
for (String chunk : chunks) {
if (StringUtils.isNotBlank(chunk)) {
chunkList.add(chunk.trim());
}
}
} else {
int indexMin = 0;
int len = content.length();
int i = 0;
int right = 0;
while (true) {
if (len > right) {
int begin = i * textBlockSize - overlapChar;
if (begin < indexMin) {
begin = indexMin;
}
int end = textBlockSize * (i + 1) + overlapChar;
if (end > len) {
end = len;
}
String chunk = content.substring(begin, end);
chunkList.add(chunk);
i++;
right = right + textBlockSize;
} else {
break;
}
}
}
return chunkList;
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, SplitterSupport::paragraphs);
}
}

38
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java
View File

@@ -1,47 +1,15 @@
package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 代码文件分片器：按空行（函数/类之间的自然边界）切分，
* 块合并到不超过块大小，超大块再按滑动窗口切分
*/
@Component
@AllArgsConstructor
@Slf4j
public class CodeTextSplitter implements TextSplitter {
private final IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content, String kid) {
int textBlockSize = 1000;
int overlapChar = 50;
if (StringUtils.isNotBlank(kid)) {
try {
KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
if (info != null) {
if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
textBlockSize = info.getTextBlockSize().intValue();
}
if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
overlapChar = info.getOverlapChar().intValue();
}
}
} catch (Exception e) {
log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
}
}
// 按空行切分，保留段落间的自然语义边界
String[] sections = content.split("\\n\\s*\\n");
return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar);
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, SplitterSupport::paragraphs);
}
}

70
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/ExcelTextSplitter.java
View File

@@ -1,79 +1,15 @@
package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class ExcelTextSplitter implements TextSplitter {
private final IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content, String kid) {
// 默认配置
String knowledgeSeparator = "#";
int textBlockSize = 1000;
int overlapChar = 50;
// 根据知识库ID查询配置，覆盖默认值
if (StringUtils.isNotBlank(kid)) {
try {
KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
if (info != null) {
if (StringUtils.isNotBlank(info.getSeparator())) {
knowledgeSeparator = info.getSeparator();
}
if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
textBlockSize = info.getTextBlockSize().intValue();
}
if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
overlapChar = info.getOverlapChar().intValue();
}
}
} catch (Exception e) {
log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
}
}
List<String> chunkList = new ArrayList<>();
if (content.contains(knowledgeSeparator) && StringUtils.isNotBlank(knowledgeSeparator)) {
// 按自定义分隔符切分
String[] chunks = content.split(knowledgeSeparator);
chunkList.addAll(Arrays.asList(chunks));
} else {
int indexMin = 0;
int len = content.length();
int i = 0;
int right = 0;
while (true) {
if (len > right) {
int begin = i * textBlockSize - overlapChar;
if (begin < indexMin) {
begin = indexMin;
}
int end = textBlockSize * (i + 1) + overlapChar;
if (end > len) {
end = len;
}
String chunk = content.substring(begin, end);
chunkList.add(chunk);
i++;
right = right + textBlockSize;
} else {
break;
}
}
}
return chunkList;
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, SplitterSupport::lines);
}
}

73
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java
View File

@@ -1,47 +1,56 @@
package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
/**
* Markdown 分片器：优先按标题（# ~ ######）切分，保持章节语义完整；
* 小节合并到不超过块大小，超大章节再按滑动窗口切分
*/
@Component
@AllArgsConstructor
@Slf4j
public class MarkdownTextSplitter implements TextSplitter {
private final IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content, String kid) {
int textBlockSize = 1000;
int overlapChar = 50;
if (StringUtils.isNotBlank(kid)) {
try {
KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
if (info != null) {
if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
textBlockSize = info.getTextBlockSize().intValue();
}
if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
overlapChar = info.getOverlapChar().intValue();
}
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, this::sections);
}
/** Split headings without treating heading-looking lines inside fenced code as headings. */
private List<String> sections(String markdown) {
String[] lines = markdown.split("\\n", -1);
List<String> sections = new ArrayList<>();
StringBuilder current = new StringBuilder();
boolean fenced = false;
String fenceMarker = null;
for (int i = 0; i < lines.length; i++) {
String line = lines[i];
String trimmed = line.stripLeading();
if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) {
String marker = trimmed.substring(0, 3);
if (!fenced) {
fenced = true;
fenceMarker = marker;
} else if (marker.equals(fenceMarker)) {
fenced = false;
fenceMarker = null;
}
} catch (Exception e) {
log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
}
boolean atx = !fenced && line.matches("^#{1,6}(\\s+.*)?$");
boolean setextTitle = !fenced && i + 1 < lines.length
&& lines[i + 1].matches("^\\s*(=+|-+)\\s*$") && !line.isBlank();
if ((atx || setextTitle) && current.length() > 0) flush(sections, current);
current.append(line);
if (i < lines.length - 1) current.append('\n');
if (setextTitle) {
current.append(lines[++i]);
if (i < lines.length - 1) current.append('\n');
}
}
// 按标题行切分（标题保留在各自小节开头）
String[] sections = content.split("(?m)(?=^#{1,6}\\s)");
return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar);
flush(sections, current);
return sections;
}
private static void flush(List<String> sections, StringBuilder current) {
if (!current.toString().isBlank()) sections.add(current.toString());
current.setLength(0);
}
}

106
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java
View File

@@ -1,71 +1,67 @@
package org.ruoyi.service.knowledge.impl.split;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import java.util.regex.Pattern;
/**
* 分片工具：提供各 Splitter 共用的滑动窗口切分与片段合并能力
*/
/** Shared, deterministic split pipeline used by every supported document format. */
public final class SplitterSupport {
private SplitterSupport() {}
private SplitterSupport() {
public static List<String> split(String content, DocumentSplitConfig config,
Function<String, List<String>> naturalSections) {
if (content == null || content.isBlank()) return List.of();
String normalized = content.replace("\r\n", "\n").replace('\r', '\n');
List<String> primary = literalSections(normalized, config.separator());
List<String> result = new ArrayList<>();
for (String part : primary) {
List<String> sections = naturalSections.apply(part);
String joined = sections.stream().filter(s -> s != null && !s.isBlank())
.map(String::strip).reduce((a, b) -> a + "\n\n" + b).orElse("");
result.addAll(slidingWindow(joined, config.blockSize(), config.overlap()));
}
return result;
}
/**
* 滑动窗口切分：每块约 blockSize 字符，相邻块保留 overlap 字符重叠
*/
static List<String> literalSections(String content, String separator) {
if (separator == null || separator.isEmpty() || !content.contains(separator)) return List.of(content);
List<String> parts = new ArrayList<>();
for (String part : content.split(Pattern.quote(separator), -1)) {
if (!part.isBlank()) parts.add(part);
}
return parts;
}
/** blockSize is a strict maximum; overlap is the repeated suffix/prefix length. */
public static List<String> slidingWindow(String content, int blockSize, int overlap) {
List<String> chunkList = new ArrayList<>();
int len = content.length();
int right = 0;
int i = 0;
while (len > right) {
int begin = i * blockSize - overlap;
if (begin < 0) {
begin = 0;
}
int end = blockSize * (i + 1) + overlap;
if (end > len) {
end = len;
}
String chunk = content.substring(begin, end).trim();
if (!chunk.isEmpty()) {
chunkList.add(chunk);
}
i++;
right = right + blockSize;
if (content == null || content.isBlank()) return List.of();
String value = content.strip();
List<String> chunks = new ArrayList<>();
int step = blockSize - overlap;
for (int start = 0; start < value.length(); start += step) {
int end = Math.min(value.length(), start + blockSize);
String chunk = value.substring(start, end).strip();
if (!chunk.isEmpty()) chunks.add(chunk);
if (end == value.length()) break;
}
return chunkList;
return chunks;
}
/**
* 将小段按顺序合并到不超过 blockSize，超过 blockSize 的单段再用滑动窗口切分
*/
/** Compatibility helper retained for callers/tests; now enforces strict maximum size. */
public static List<String> mergeAndSplit(String[] sections, int blockSize, int overlap) {
List<String> chunkList = new ArrayList<>();
StringBuilder current = new StringBuilder();
for (String section : sections) {
if (section == null || section.isBlank()) {
continue;
}
if (section.length() > blockSize) {
// 超长段先冲刷当前缓冲，再单独窗口切分
if (current.length() > 0) {
chunkList.add(current.toString().trim());
current.setLength(0);
}
chunkList.addAll(slidingWindow(section, blockSize, overlap));
} else if (current.length() + section.length() > blockSize) {
chunkList.add(current.toString().trim());
current.setLength(0);
current.append(section);
} else {
current.append(section);
}
}
if (current.length() > 0) {
chunkList.add(current.toString().trim());
}
return chunkList;
return split(String.join("\n\n", sections),
new DocumentSplitConfig(null, blockSize, overlap, ""),
text -> List.of(text));
}
static List<String> paragraphs(String content) {
return List.of(content.split("\\n\\s*\\n+"));
}
static List<String> lines(String content) {
return List.of(content.split("\\n+"));
}
}

5
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/TokenTextSplitter.java
View File

@@ -3,6 +3,7 @@ package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.service.knowledge.TextSplitter;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.springframework.stereotype.Component;
import java.util.List;
@@ -12,7 +13,7 @@ import java.util.List;
@Slf4j
public class TokenTextSplitter implements TextSplitter {
@Override
public List<String> split(String content, String kid) {
return null;
public List<String> split(String content, DocumentSplitConfig config) {
return SplitterSupport.split(content, config, SplitterSupport::paragraphs);
}
}

98
ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java
View File

@@ -1,78 +1,72 @@
package org.ruoyi.service.knowledge.impl.split;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Tag;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.junit.jupiter.api.Test;
import org.ruoyi.common.core.exception.ServiceException;
import org.ruoyi.factory.ResourceLoaderFactory;
import org.ruoyi.service.knowledge.DocumentSplitConfig;
import org.ruoyi.service.knowledge.impl.loader.CodeFileLoader;
import org.ruoyi.service.knowledge.impl.loader.MarkDownFileLoader;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@Tag("dev")
class RagTextSplitterRegressionTest {
private static IKnowledgeInfoService knowledgeService(String separator, long blockSize, long overlap) {
IKnowledgeInfoService service = mock(IKnowledgeInfoService.class);
KnowledgeInfoVo info = new KnowledgeInfoVo();
info.setSeparator(separator);
info.setTextBlockSize(blockSize);
info.setOverlapChar(overlap);
when(service.queryById(1L)).thenReturn(info);
return service;
@Test
void allSplittersHonorLiteralSeparatorAndStrictMaximum() {
List.of(new CharacterTextSplitter(), new MarkdownTextSplitter(),
new CodeTextSplitter(), new ExcelTextSplitter()).forEach(splitter -> {
for (String separator : List.of("|", ".", "*", "<CUT>", "\n---\n")) {
String content = "alpha" + separator + "b".repeat(35);
List<String> chunks = splitter.split(content, config(separator, 12, 0));
assertTrue(chunks.size() >= 4, splitter.getClass().getSimpleName());
assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 12));
assertTrue(chunks.stream().noneMatch(chunk -> chunk.contains(separator)));
}
});
}
@Test
void characterSplitterTreatsRegexMetacharactersLiterally() {
CharacterTextSplitter pipe = new CharacterTextSplitter(knowledgeService("|", 1000, 50));
assertEquals(List.of("alpha", "beta", "gamma"), pipe.split("alpha|beta|gamma", "1"));
CharacterTextSplitter dot = new CharacterTextSplitter(knowledgeService(".", 1000, 50));
assertEquals(List.of("alpha", "beta", "gamma"), dot.split("alpha.beta.gamma", "1"));
CharacterTextSplitter star = new CharacterTextSplitter(knowledgeService("*", 1000, 50));
assertEquals(List.of("alpha", "beta", "gamma"), star.split("alpha*beta*gamma", "1"));
void zeroOverlapIsRespectedAndConfiguredOverlapIsExact() {
CharacterTextSplitter splitter = new CharacterTextSplitter();
assertEquals(List.of("abcdefghij", "klmnopqrst"),
splitter.split("abcdefghijklmnopqrst", config(null, 10, 0)));
List<String> overlap = splitter.split("abcdefghijklmnop", config(null, 10, 3));
assertEquals("hij", overlap.get(0).substring(7));
assertTrue(overlap.get(1).startsWith("hij"));
assertTrue(overlap.stream().allMatch(chunk -> chunk.length() <= 10));
}
@Test
void markdownSplitterReturnsNonEmptyBoundedChunks() {
MarkdownTextSplitter splitter = new MarkdownTextSplitter(knowledgeService(null, 40, 5));
String markdown = "# Title\nintro text\n## Details\n" + "detail ".repeat(20);
List<String> chunks = splitter.split(markdown, "1");
assertFalse(chunks.isEmpty());
assertTrue(chunks.stream().noneMatch(String::isBlank));
assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 50),
"window size may include overlap on both sides");
assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("# Title")));
void markdownKeepsHeadingsAndIgnoresHeadingsInsideFences() {
MarkdownTextSplitter splitter = new MarkdownTextSplitter();
String markdown = "# Real\nintro\n```md\n# not a heading\n```\nTitle\n=====\nbody";
List<String> chunks = splitter.split(markdown, config(null, 200, 0));
assertEquals(1, chunks.size());
assertTrue(chunks.get(0).contains("# Real"));
assertTrue(chunks.get(0).contains("# not a heading"));
assertTrue(chunks.get(0).contains("Title\n====="));
}
@Test
void codeSplitterReturnsNonEmptyChunksAndPreservesContent() {
CodeTextSplitter splitter = new CodeTextSplitter(knowledgeService(null, 45, 5));
String code = "class A {\n void a() {}\n}\n\nclass B {\n" + " int value = 1;\n".repeat(8) + "}";
List<String> chunks = splitter.split(code, "1");
assertFalse(chunks.isEmpty());
assertTrue(chunks.stream().noneMatch(String::isBlank));
assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class A")));
assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class B") || chunk.contains("int value")));
void invalidConfigurationFailsClearly() {
assertThrows(ServiceException.class, () -> config(null, 0, 0));
assertThrows(ServiceException.class, () -> config(null, 10, 10));
assertThrows(ServiceException.class, () -> config(null, 10, -1));
}
@Test
void splitterSupportHandlesEmptyAndOversizedSections() {
assertTrue(SplitterSupport.mergeAndSplit(new String[]{"", " "}, 20, 3).isEmpty());
void loaderFactoryNormalizesSuffixAndUsesFormatSpecificLoader() {
ResourceLoaderFactory factory = new ResourceLoaderFactory(new CharacterTextSplitter(),
new CodeTextSplitter(), new MarkdownTextSplitter(), new ExcelTextSplitter());
assertInstanceOf(MarkDownFileLoader.class, factory.getLoaderByFileType(" .MD "));
assertInstanceOf(CodeFileLoader.class, factory.getLoaderByFileType(".JAVA"));
}
List<String> chunks = SplitterSupport.mergeAndSplit(
new String[]{"short", "x".repeat(55)}, 20, 3);
assertEquals("short", chunks.get(0));
assertTrue(chunks.size() >= 4);
assertTrue(chunks.stream().noneMatch(String::isBlank));
assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 26));
private DocumentSplitConfig config(String separator, int size, int overlap) {
return new DocumentSplitConfig(separator, size, overlap, "md");
}
}