From a6a55202a355fc0bb429d62fa794db104000ae30 Mon Sep 17 00:00:00 2001 From: evo <446796145@qq.com> Date: Tue, 21 Jul 2026 09:34:47 +0800 Subject: [PATCH] =?UTF-8?q?fix(rag):=20=E4=BF=AE=E5=A4=8D=E6=96=87?= =?UTF-8?q?=E6=9C=AC=E5=88=86=E7=89=87=E8=BE=B9=E7=95=8C=E4=B8=8E=E5=AD=97?= =?UTF-8?q?=E9=9D=A2=E9=87=8F=E5=88=86=E9=9A=94=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/split/CharacterTextSplitter.java | 12 ++- .../impl/split/CodeTextSplitter.java | 31 +++++++- .../impl/split/MarkdownTextSplitter.java | 31 +++++++- .../knowledge/impl/split/SplitterSupport.java | 71 +++++++++++++++++ .../split/RagTextSplitterRegressionTest.java | 78 +++++++++++++++++++ 5 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java create mode 100644 ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java index 107f0dc1..950608f6 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java @@ -49,10 +49,14 @@ public class CharacterTextSplitter implements TextSplitter { } List chunkList = new ArrayList<>(); - if (content.contains(knowledgeSeparator) && StringUtils.isNotBlank(knowledgeSeparator)) { - // 按自定义分隔符切分 - String[] chunks = content.split(knowledgeSeparator); - chunkList.addAll(Arrays.asList(chunks)); + if (StringUtils.isNotBlank(knowledgeSeparator) && content.contains(knowledgeSeparator)) { + // 按自定义分隔符切分（字面量匹配，避免分隔符被当作正则） + String[] chunks = content.split(java.util.regex.Pattern.quote(knowledgeSeparator)); + for (String chunk : chunks) { + if (StringUtils.isNotBlank(chunk)) { + chunkList.add(chunk.trim()); + } + } } else { int indexMin = 0; int len = content.length(); diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java index 5cba55c5..98badf0b 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java @@ -2,17 +2,46 @@ package org.ruoyi.service.knowledge.impl.split; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.ruoyi.common.core.utils.StringUtils; +import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; +import org.ruoyi.service.knowledge.IKnowledgeInfoService; import org.ruoyi.service.knowledge.TextSplitter; import org.springframework.stereotype.Component; import java.util.List; +/** + * 代码文件分片器：按空行（函数/类之间的自然边界）切分， + * 块合并到不超过块大小，超大块再按滑动窗口切分 + */ @Component @AllArgsConstructor @Slf4j public class CodeTextSplitter implements TextSplitter { + + private final IKnowledgeInfoService knowledgeInfoService; + @Override public List split(String content, String kid) { - return null; + int textBlockSize = 1000; + int overlapChar = 50; + if (StringUtils.isNotBlank(kid)) { + try { + KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid)); + if (info != null) { + if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) { + textBlockSize = info.getTextBlockSize().intValue(); + } + if (info.getOverlapChar() != null && info.getOverlapChar() > 0) { + overlapChar = info.getOverlapChar().intValue(); + } + } + } catch (Exception e) { + log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e); + } + } + // 按空行切分，保留段落间的自然语义边界 + String[] sections = content.split("\\n\\s*\\n"); + return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java index a371273d..38358749 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java @@ -2,17 +2,46 @@ package org.ruoyi.service.knowledge.impl.split; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.ruoyi.common.core.utils.StringUtils; +import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; +import org.ruoyi.service.knowledge.IKnowledgeInfoService; import org.ruoyi.service.knowledge.TextSplitter; import org.springframework.stereotype.Component; import java.util.List; +/** + * Markdown 分片器：优先按标题（# ~ ######）切分，保持章节语义完整； + * 小节合并到不超过块大小，超大章节再按滑动窗口切分 + */ @Component @AllArgsConstructor @Slf4j public class MarkdownTextSplitter implements TextSplitter { + + private final IKnowledgeInfoService knowledgeInfoService; + @Override public List split(String content, String kid) { - return null; + int textBlockSize = 1000; + int overlapChar = 50; + if (StringUtils.isNotBlank(kid)) { + try { + KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid)); + if (info != null) { + if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) { + textBlockSize = info.getTextBlockSize().intValue(); + } + if (info.getOverlapChar() != null && info.getOverlapChar() > 0) { + overlapChar = info.getOverlapChar().intValue(); + } + } + } catch (Exception e) { + log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e); + } + } + // 按标题行切分（标题保留在各自小节开头） + String[] sections = content.split("(?m)(?=^#{1,6}\\s)"); + return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java new file mode 100644 index 00000000..dbf91360 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java @@ -0,0 +1,71 @@ +package org.ruoyi.service.knowledge.impl.split; + +import java.util.ArrayList; +import java.util.List; + +/** + * 分片工具：提供各 Splitter 共用的滑动窗口切分与片段合并能力 + */ +public final class SplitterSupport { + + private SplitterSupport() { + } + + /** + * 滑动窗口切分：每块约 blockSize 字符，相邻块保留 overlap 字符重叠 + */ + public static List slidingWindow(String content, int blockSize, int overlap) { + List chunkList = new ArrayList<>(); + int len = content.length(); + int right = 0; + int i = 0; + while (len > right) { + int begin = i * blockSize - overlap; + if (begin < 0) { + begin = 0; + } + int end = blockSize * (i + 1) + overlap; + if (end > len) { + end = len; + } + String chunk = content.substring(begin, end).trim(); + if (!chunk.isEmpty()) { + chunkList.add(chunk); + } + i++; + right = right + blockSize; + } + return chunkList; + } + + /** + * 将小段按顺序合并到不超过 blockSize，超过 blockSize 的单段再用滑动窗口切分 + */ + public static List mergeAndSplit(String[] sections, int blockSize, int overlap) { + List chunkList = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + for (String section : sections) { + if (section == null || section.isBlank()) { + continue; + } + if (section.length() > blockSize) { + // 超长段先冲刷当前缓冲，再单独窗口切分 + if (current.length() > 0) { + chunkList.add(current.toString().trim()); + current.setLength(0); + } + chunkList.addAll(slidingWindow(section, blockSize, overlap)); + } else if (current.length() + section.length() > blockSize) { + chunkList.add(current.toString().trim()); + current.setLength(0); + current.append(section); + } else { + current.append(section); + } + } + if (current.length() > 0) { + chunkList.add(current.toString().trim()); + } + return chunkList; + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java b/ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java new file mode 100644 index 00000000..b3f2e151 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java @@ -0,0 +1,78 @@ +package org.ruoyi.service.knowledge.impl.split; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Tag; +import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; +import org.ruoyi.service.knowledge.IKnowledgeInfoService; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Tag("dev") +class RagTextSplitterRegressionTest { + + private static IKnowledgeInfoService knowledgeService(String separator, long blockSize, long overlap) { + IKnowledgeInfoService service = mock(IKnowledgeInfoService.class); + KnowledgeInfoVo info = new KnowledgeInfoVo(); + info.setSeparator(separator); + info.setTextBlockSize(blockSize); + info.setOverlapChar(overlap); + when(service.queryById(1L)).thenReturn(info); + return service; + } + + @Test + void characterSplitterTreatsRegexMetacharactersLiterally() { + CharacterTextSplitter pipe = new CharacterTextSplitter(knowledgeService("|", 1000, 50)); + assertEquals(List.of("alpha", "beta", "gamma"), pipe.split("alpha|beta|gamma", "1")); + + CharacterTextSplitter dot = new CharacterTextSplitter(knowledgeService(".", 1000, 50)); + assertEquals(List.of("alpha", "beta", "gamma"), dot.split("alpha.beta.gamma", "1")); + + CharacterTextSplitter star = new CharacterTextSplitter(knowledgeService("*", 1000, 50)); + assertEquals(List.of("alpha", "beta", "gamma"), star.split("alpha*beta*gamma", "1")); + } + + @Test + void markdownSplitterReturnsNonEmptyBoundedChunks() { + MarkdownTextSplitter splitter = new MarkdownTextSplitter(knowledgeService(null, 40, 5)); + String markdown = "# Title\nintro text\n## Details\n" + "detail ".repeat(20); + + List chunks = splitter.split(markdown, "1"); + + assertFalse(chunks.isEmpty()); + assertTrue(chunks.stream().noneMatch(String::isBlank)); + assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 50), + "window size may include overlap on both sides"); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("# Title"))); + } + + @Test + void codeSplitterReturnsNonEmptyChunksAndPreservesContent() { + CodeTextSplitter splitter = new CodeTextSplitter(knowledgeService(null, 45, 5)); + String code = "class A {\n void a() {}\n}\n\nclass B {\n" + " int value = 1;\n".repeat(8) + "}"; + + List chunks = splitter.split(code, "1"); + + assertFalse(chunks.isEmpty()); + assertTrue(chunks.stream().noneMatch(String::isBlank)); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class A"))); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class B") || chunk.contains("int value"))); + } + + @Test + void splitterSupportHandlesEmptyAndOversizedSections() { + assertTrue(SplitterSupport.mergeAndSplit(new String[]{"", " "}, 20, 3).isEmpty()); + + List chunks = SplitterSupport.mergeAndSplit( + new String[]{"short", "x".repeat(55)}, 20, 3); + + assertEquals("short", chunks.get(0)); + assertTrue(chunks.size() >= 4); + assertTrue(chunks.stream().noneMatch(String::isBlank)); + assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 26)); + } +}