xieyinlu/ruoyi-ai
1
0
Fork 0
mirror of https://gitcode.com/ageerle/ruoyi-ai.git synced 2026-09-13 00:14:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(rag): 修复文本分片边界与字面量分隔符

Browse Source
This commit is contained in:
evo
2026-07-21 09:34:47 +08:00
parent 97ce15d116
commit a6a55202a3
5 changed files with 217 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java
View File

@@ -49,10 +49,14 @@ public class CharacterTextSplitter implements TextSplitter {
}
List<String> chunkList = new ArrayList<>();
if (content.contains(knowledgeSeparator) && StringUtils.isNotBlank(knowledgeSeparator)) {
// 按自定义分隔符切分
String[] chunks = content.split(knowledgeSeparator);
chunkList.addAll(Arrays.asList(chunks));
if (StringUtils.isNotBlank(knowledgeSeparator) && content.contains(knowledgeSeparator)) {
// 按自定义分隔符切分（字面量匹配，避免分隔符被当作正则）
String[] chunks = content.split(java.util.regex.Pattern.quote(knowledgeSeparator));
for (String chunk : chunks) {
if (StringUtils.isNotBlank(chunk)) {
chunkList.add(chunk.trim());
}
}
} else {
int indexMin = 0;
int len = content.length();

31
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java
View File

@@ -2,17 +2,46 @@ package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 代码文件分片器：按空行（函数/类之间的自然边界）切分，
* 块合并到不超过块大小，超大块再按滑动窗口切分
*/
@Component
@AllArgsConstructor
@Slf4j
public class CodeTextSplitter implements TextSplitter {
private final IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content, String kid) {
return null;
int textBlockSize = 1000;
int overlapChar = 50;
if (StringUtils.isNotBlank(kid)) {
try {
KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
if (info != null) {
if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
textBlockSize = info.getTextBlockSize().intValue();
}
if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
overlapChar = info.getOverlapChar().intValue();
}
}
} catch (Exception e) {
log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
}
}
// 按空行切分，保留段落间的自然语义边界
String[] sections = content.split("\\n\\s*\\n");
return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar);
}
}

31
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java
View File

@@ -2,17 +2,46 @@ package org.ruoyi.service.knowledge.impl.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import org.ruoyi.service.knowledge.TextSplitter;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* Markdown 分片器：优先按标题（# ~ ######）切分，保持章节语义完整；
* 小节合并到不超过块大小，超大章节再按滑动窗口切分
*/
@Component
@AllArgsConstructor
@Slf4j
public class MarkdownTextSplitter implements TextSplitter {
private final IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content, String kid) {
return null;
int textBlockSize = 1000;
int overlapChar = 50;
if (StringUtils.isNotBlank(kid)) {
try {
KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
if (info != null) {
if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
textBlockSize = info.getTextBlockSize().intValue();
}
if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
overlapChar = info.getOverlapChar().intValue();
}
}
} catch (Exception e) {
log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
}
}
// 按标题行切分（标题保留在各自小节开头）
String[] sections = content.split("(?m)(?=^#{1,6}\\s)");
return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar);
}
}

71
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java Normal file
View File

@@ -0,0 +1,71 @@
package org.ruoyi.service.knowledge.impl.split;
import java.util.ArrayList;
import java.util.List;
/**
* 分片工具：提供各 Splitter 共用的滑动窗口切分与片段合并能力
*/
public final class SplitterSupport {
private SplitterSupport() {
}
/**
* 滑动窗口切分：每块约 blockSize 字符，相邻块保留 overlap 字符重叠
*/
public static List<String> slidingWindow(String content, int blockSize, int overlap) {
List<String> chunkList = new ArrayList<>();
int len = content.length();
int right = 0;
int i = 0;
while (len > right) {
int begin = i * blockSize - overlap;
if (begin < 0) {
begin = 0;
}
int end = blockSize * (i + 1) + overlap;
if (end > len) {
end = len;
}
String chunk = content.substring(begin, end).trim();
if (!chunk.isEmpty()) {
chunkList.add(chunk);
}
i++;
right = right + blockSize;
}
return chunkList;
}
/**
* 将小段按顺序合并到不超过 blockSize，超过 blockSize 的单段再用滑动窗口切分
*/
public static List<String> mergeAndSplit(String[] sections, int blockSize, int overlap) {
List<String> chunkList = new ArrayList<>();
StringBuilder current = new StringBuilder();
for (String section : sections) {
if (section == null || section.isBlank()) {
continue;
}
if (section.length() > blockSize) {
// 超长段先冲刷当前缓冲，再单独窗口切分
if (current.length() > 0) {
chunkList.add(current.toString().trim());
current.setLength(0);
}
chunkList.addAll(slidingWindow(section, blockSize, overlap));
} else if (current.length() + section.length() > blockSize) {
chunkList.add(current.toString().trim());
current.setLength(0);
current.append(section);
} else {
current.append(section);
}
}
if (current.length() > 0) {
chunkList.add(current.toString().trim());
}
return chunkList;
}
}

78
ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java Normal file
View File

@@ -0,0 +1,78 @@
package org.ruoyi.service.knowledge.impl.split;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Tag;
import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
import org.ruoyi.service.knowledge.IKnowledgeInfoService;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@Tag("dev")
class RagTextSplitterRegressionTest {
private static IKnowledgeInfoService knowledgeService(String separator, long blockSize, long overlap) {
IKnowledgeInfoService service = mock(IKnowledgeInfoService.class);
KnowledgeInfoVo info = new KnowledgeInfoVo();
info.setSeparator(separator);
info.setTextBlockSize(blockSize);
info.setOverlapChar(overlap);
when(service.queryById(1L)).thenReturn(info);
return service;
}
@Test
void characterSplitterTreatsRegexMetacharactersLiterally() {
CharacterTextSplitter pipe = new CharacterTextSplitter(knowledgeService("|", 1000, 50));
assertEquals(List.of("alpha", "beta", "gamma"), pipe.split("alpha|beta|gamma", "1"));
CharacterTextSplitter dot = new CharacterTextSplitter(knowledgeService(".", 1000, 50));
assertEquals(List.of("alpha", "beta", "gamma"), dot.split("alpha.beta.gamma", "1"));
CharacterTextSplitter star = new CharacterTextSplitter(knowledgeService("*", 1000, 50));
assertEquals(List.of("alpha", "beta", "gamma"), star.split("alpha*beta*gamma", "1"));
}
@Test
void markdownSplitterReturnsNonEmptyBoundedChunks() {
MarkdownTextSplitter splitter = new MarkdownTextSplitter(knowledgeService(null, 40, 5));
String markdown = "# Title\nintro text\n## Details\n" + "detail ".repeat(20);
List<String> chunks = splitter.split(markdown, "1");
assertFalse(chunks.isEmpty());
assertTrue(chunks.stream().noneMatch(String::isBlank));
assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 50),
"window size may include overlap on both sides");
assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("# Title")));
}
@Test
void codeSplitterReturnsNonEmptyChunksAndPreservesContent() {
CodeTextSplitter splitter = new CodeTextSplitter(knowledgeService(null, 45, 5));
String code = "class A {\n void a() {}\n}\n\nclass B {\n" + " int value = 1;\n".repeat(8) + "}";
List<String> chunks = splitter.split(code, "1");
assertFalse(chunks.isEmpty());
assertTrue(chunks.stream().noneMatch(String::isBlank));
assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class A")));
assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class B") || chunk.contains("int value")));
}
@Test
void splitterSupportHandlesEmptyAndOversizedSections() {
assertTrue(SplitterSupport.mergeAndSplit(new String[]{"", " "}, 20, 3).isEmpty());
List<String> chunks = SplitterSupport.mergeAndSplit(
new String[]{"short", "x".repeat(55)}, 20, 3);
assertEquals("short", chunks.get(0));
assertTrue(chunks.size() >= 4);
assertTrue(chunks.stream().noneMatch(String::isBlank));
assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 26));
}
}