fix(rag): 统一文档切割配置并支持重新解析

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/knowledge/KnowledgeAttachController.java

@@ -9,6 +9,7 @@ import cn.dev33.satoken.annotation.SaCheckPermission;
 import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo;
 import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo;
 import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
+import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
 import org.ruoyi.service.knowledge.IKnowledgeAttachService;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.validation.annotation.Validated;
@@ -129,4 +130,12 @@ public class KnowledgeAttachController extends BaseController {
         knowledgeAttachService.parse(id);
         return R.ok();
     }
+
+    @SaCheckPermission("system:attach:edit")
+    @Log(title = "知识库附件批量重新解析", businessType = BusinessType.UPDATE)
+    @PostMapping("/reparse/knowledge/{knowledgeId}")
+    @RepeatSubmit()
+    public R<KnowledgeReparseVo> reparseKnowledge(@PathVariable Long knowledgeId) {
+        return R.ok(knowledgeAttachService.reparseKnowledge(knowledgeId));
+    }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/knowledge/KnowledgeReparseVo.java

@@ -0,0 +1,4 @@
+package org.ruoyi.domain.vo.knowledge;
+
+public record KnowledgeReparseVo(int submitted, int skipped, int total) {
+}

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/ResourceLoaderFactory.java

@@ -17,7 +17,7 @@ public class ResourceLoaderFactory {
     private final ExcelTextSplitter excelTextSplitter;
 
     public ResourceLoader getLoaderByFileType(String fileType) {
-        fileType = StringUtils.removeStart(fileType, ".");
+        fileType = StringUtils.lowerCase(StringUtils.removeStart(StringUtils.trim(fileType), "."));
         if (FileTypeConstants.isTextFile(fileType)) {
             return new TextFileLoader(characterTextSplitter);
         } else if (FileTypeConstants.isWord(fileType)) {

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/DocumentSplitConfig.java

@@ -0,0 +1,20 @@
+package org.ruoyi.service.knowledge;
+
+import org.ruoyi.common.core.exception.ServiceException;
+
+/** Immutable snapshot of the split settings used for one parse operation. */
+public record DocumentSplitConfig(String separator, int blockSize, int overlap, String fileType) {
+
+    public static final int DEFAULT_BLOCK_SIZE = 1000;
+    public static final int DEFAULT_OVERLAP = 50;
+
+    public DocumentSplitConfig {
+        if (blockSize <= 0) {
+            throw new ServiceException("文本块大小必须大于0");
+        }
+        if (overlap < 0 || overlap >= blockSize) {
+            throw new ServiceException("重叠字符数必须大于等于0且小于文本块大小");
+        }
+        fileType = fileType == null ? "" : fileType.strip().replaceFirst("^\\.", "").toLowerCase();
+    }
+}

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/IKnowledgeAttachService.java

@@ -5,6 +5,7 @@ import org.ruoyi.common.mybatis.core.page.PageQuery;
 import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo;
 import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo;
 import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
+import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
 
 import java.util.Collection;
 import java.util.List;
@@ -79,4 +80,6 @@ public interface IKnowledgeAttachService {
      * @param id 附件ID
      */
     void parse(Long id);
+
+    KnowledgeReparseVo reparseKnowledge(Long knowledgeId);
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/ResourceLoader.java

@@ -10,5 +10,5 @@ public interface ResourceLoader {
 
     String getContent(InputStream inputStream);
 
-    List<String> getChunkList(String content, String kid);
+    List<String> getChunkList(String content, DocumentSplitConfig config);
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/TextSplitter.java

@@ -11,8 +11,8 @@ public interface TextSplitter {
      * 文本切分
      *
      * @param content 文本内容
-     * @param kid     知识库id
+     * @param config  本次解析的分片配置快照
      * @return 切分后的文本列表
      */
-    List<String> split(String content, String kid);
+    List<String> split(String content, DocumentSplitConfig config);
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java

@@ -27,12 +27,14 @@ import org.ruoyi.domain.entity.knowledge.KnowledgeFragment;
 import org.ruoyi.domain.vo.knowledge.DocFragmentCountVo;
 import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo;
 import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
+import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo;
 import org.ruoyi.factory.ResourceLoaderFactory;
 import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper;
 import org.ruoyi.mapper.knowledge.KnowledgeFragmentMapper;
 import org.ruoyi.service.knowledge.IKnowledgeAttachService;
 import org.ruoyi.service.knowledge.IKnowledgeInfoService;
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 import org.ruoyi.service.vector.VectorStoreService;
 import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
 import org.springframework.scheduling.annotation.Async;
@@ -184,10 +186,17 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
     @Override
     public void parse(Long id) {
         KnowledgeAttach attach = baseMapper.selectById(id);
-        if (attach == null || (!KnowledgeAttachStatus.WAITING.getCode().equals(attach.getStatus()) && !KnowledgeAttachStatus.FAILED.getCode().equals(attach.getStatus()))) {
+        if (attach == null || KnowledgeAttachStatus.PARSING.getCode().equals(attach.getStatus())) {
             return;
         }
 
+        int claimed = baseMapper.update(null, Wrappers.<KnowledgeAttach>lambdaUpdate()
+            .set(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode())
+            .set(KnowledgeAttach::getRemark, null)
+            .eq(KnowledgeAttach::getId, id)
+            .ne(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode()));
+        if (claimed == 0) return;
+
         try {
             attach.setStatus(KnowledgeAttachStatus.PARSING.getCode()); // 解析中
             baseMapper.updateById(attach);
@@ -196,6 +205,16 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
 
             Long knowledgeId = attach.getKnowledgeId();
             String docId = attach.getDocId();
+            KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
+            if (knowledgeInfoVo == null) {
+                throw new ServiceException("知识库不存在: " + knowledgeId);
+            }
+            int blockSize = knowledgeInfoVo.getTextBlockSize() == null
+                ? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : knowledgeInfoVo.getTextBlockSize().intValue();
+            int overlap = knowledgeInfoVo.getOverlapChar() == null
+                ? DocumentSplitConfig.DEFAULT_OVERLAP : knowledgeInfoVo.getOverlapChar().intValue();
+            DocumentSplitConfig splitConfig = new DocumentSplitConfig(
+                knowledgeInfoVo.getSeparator(), blockSize, overlap, attach.getType());
             List<KnowledgeFragment> oldFragments = knowledgeFragmentMapper.selectList(
                     Wrappers.<KnowledgeFragment>lambdaQuery().eq(KnowledgeFragment::getDocId, docId));
 
@@ -210,7 +229,7 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
             try (InputStream inputStream = new URL(ossDTO.getUrl()).openStream()) {
                 content = resourceLoader.getContent(inputStream);
             }
-            List<String> chunkList = resourceLoader.getChunkList(content, String.valueOf(knowledgeId));
+            List<String> chunkList = resourceLoader.getChunkList(content, splitConfig);
 
             if (CollUtil.isEmpty(chunkList)) {
                 throw new RuntimeException("文档分片结果为空，请检查文档内容或分片器是否支持该文件类型");
@@ -231,7 +250,6 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
                 knowledgeFragment.setCreateTime(new Date());
                 knowledgeFragmentList.add(knowledgeFragment);
             }
-            KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId);
             ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel());
 
             StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo();
@@ -275,4 +293,22 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService {
             baseMapper.updateById(attach);
         }
     }
+
+    @Override
+    public KnowledgeReparseVo reparseKnowledge(Long knowledgeId) {
+        List<KnowledgeAttach> attachments = baseMapper.selectList(
+            Wrappers.<KnowledgeAttach>lambdaQuery().eq(KnowledgeAttach::getKnowledgeId, knowledgeId));
+        int submitted = 0;
+        int skipped = 0;
+        IKnowledgeAttachService proxy = SpringUtils.getBean(IKnowledgeAttachService.class);
+        for (KnowledgeAttach attachment : attachments) {
+            if (KnowledgeAttachStatus.PARSING.getCode().equals(attachment.getStatus())) {
+                skipped++;
+            } else {
+                proxy.parse(attachment.getId());
+                submitted++;
+            }
+        }
+        return new KnowledgeReparseVo(submitted, skipped, attachments.size());
+    }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeInfoServiceImpl.java

@@ -18,6 +18,7 @@ import org.ruoyi.service.knowledge.IKnowledgeInfoService;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.ruoyi.service.retrieval.KnowledgeRetrievalService;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 import org.ruoyi.common.core.service.OssService;
 
 import java.util.List;
@@ -152,6 +153,11 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService {
      * 保存前的数据校验
      */
     private void validEntityBeforeSave(KnowledgeInfo entity){
+        int blockSize = entity.getTextBlockSize() == null
+            ? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : entity.getTextBlockSize().intValue();
+        int overlap = entity.getOverlapChar() == null
+            ? DocumentSplitConfig.DEFAULT_OVERLAP : entity.getOverlapChar().intValue();
+        new DocumentSplitConfig(entity.getSeparator(), blockSize, overlap, "");
         //TODO 做一些数据校验,如唯一约束
     }
 

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CodeFileLoader.java

@@ -3,13 +3,16 @@ package org.ruoyi.service.knowledge.impl.loader;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 import org.ruoyi.service.knowledge.TextSplitter;
+import org.ruoyi.common.core.exception.ServiceException;
 import org.springframework.stereotype.Component;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 @Component
@@ -21,20 +24,20 @@ public class CodeFileLoader implements ResourceLoader {
     @Override
     public String getContent(InputStream inputStream) {
         StringBuffer stringBuffer = new StringBuffer();
-        try (InputStreamReader reader = new InputStreamReader(inputStream);
+        try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
              BufferedReader bufferedReader = new BufferedReader(reader)) {
             String line;
             while ((line = bufferedReader.readLine()) != null) {
                 stringBuffer.append(line).append("\n");
             }
         } catch (IOException e) {
-            e.printStackTrace();
+            throw new ServiceException("读取代码文件失败", e);
         }
         return stringBuffer.toString();
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
-        return textSplitter.split(content, kid);
+        return textSplitter.split(content, config);
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CsvFileLoader.java

@@ -1,6 +1,7 @@
 package org.ruoyi.service.knowledge.impl.loader;
 
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 
 import java.io.InputStream;
 import java.util.List;
@@ -12,7 +13,7 @@ public class CsvFileLoader implements ResourceLoader {
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
         return null;
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/ExcelFileLoader.java

@@ -6,6 +6,7 @@ import dev.langchain4j.data.document.parser.apache.tika.ApacheTikaDocumentParser
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 import org.ruoyi.service.knowledge.TextSplitter;
 import org.springframework.stereotype.Component;
 
@@ -38,7 +39,7 @@ public class ExcelFileLoader implements ResourceLoader {
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
-        return textSplitter.split(content, kid);
+        return textSplitter.split(content, config);
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/FolderLoader.java

@@ -1,6 +1,7 @@
 package org.ruoyi.service.knowledge.impl.loader;
 
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 
 import java.io.InputStream;
 import java.util.List;
@@ -12,7 +13,7 @@ public class FolderLoader implements ResourceLoader {
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
         return null;
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/GithubLoader.java

@@ -1,6 +1,7 @@
 package org.ruoyi.service.knowledge.impl.loader;
 
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 
 import java.io.InputStream;
 import java.util.List;
@@ -12,7 +13,7 @@ public class GithubLoader implements ResourceLoader {
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
         return null;
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/JsonFileLoader.java

@@ -1,6 +1,7 @@
 package org.ruoyi.service.knowledge.impl.loader;
 
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 
 import java.io.InputStream;
 import java.util.List;
@@ -12,7 +13,7 @@ public class JsonFileLoader implements ResourceLoader {
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
         return null;
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/MarkDownFileLoader.java

@@ -3,13 +3,16 @@ package org.ruoyi.service.knowledge.impl.loader;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 import org.ruoyi.service.knowledge.TextSplitter;
+import org.ruoyi.common.core.exception.ServiceException;
 import org.springframework.stereotype.Component;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 @Component
@@ -21,20 +24,20 @@ public class MarkDownFileLoader implements ResourceLoader {
     @Override
     public String getContent(InputStream inputStream) {
         StringBuffer stringBuffer = new StringBuffer();
-        try (InputStreamReader reader = new InputStreamReader(inputStream);
+        try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
              BufferedReader bufferedReader = new BufferedReader(reader)) {
             String line;
             while ((line = bufferedReader.readLine()) != null) {
                 stringBuffer.append(line).append("\n");
             }
         } catch (IOException e) {
-            e.printStackTrace();
+            throw new ServiceException("读取 Markdown 文件失败", e);
         }
         return stringBuffer.toString();
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
-        return textSplitter.split(content, kid);
+        return textSplitter.split(content, config);
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/PdfFileLoader.java

@@ -6,6 +6,7 @@ import org.apache.pdfbox.io.RandomAccessReadBuffer;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.text.PDFTextStripper;
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 import org.ruoyi.service.knowledge.TextSplitter;
 import org.springframework.stereotype.Component;
 
@@ -32,7 +33,7 @@ public class PdfFileLoader implements ResourceLoader {
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
-        return characterTextSplitter.split(content, kid);
+        return characterTextSplitter.split(content, config);
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/TextFileLoader.java

@@ -3,7 +3,9 @@ package org.ruoyi.service.knowledge.impl.loader;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 import org.ruoyi.service.knowledge.TextSplitter;
+import org.ruoyi.common.core.exception.ServiceException;
 import org.springframework.stereotype.Component;
 
 import java.io.BufferedReader;
@@ -22,18 +24,16 @@ public class TextFileLoader implements ResourceLoader {
 
     @Override
     public String getContent(InputStream inputStream) {
-        String stringBuffer = "";
         try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
              BufferedReader bufferedReader = new BufferedReader(reader)) {
-            stringBuffer = bufferedReader.lines().collect(Collectors.joining());
+            return bufferedReader.lines().collect(Collectors.joining("\n"));
         } catch (IOException e) {
-            e.printStackTrace();
+            throw new ServiceException("读取文本文件失败", e);
         }
-        return stringBuffer;
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
-        return textSplitter.split(content, kid);
+        return textSplitter.split(content, config);
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/WordLoader.java

@@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
 import org.apache.poi.xwpf.usermodel.XWPFDocument;
 import org.ruoyi.service.knowledge.ResourceLoader;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 import org.ruoyi.service.knowledge.TextSplitter;
 import org.springframework.stereotype.Component;
 
@@ -32,8 +33,8 @@ public class WordLoader implements ResourceLoader {
     }
 
     @Override
-    public List<String> getChunkList(String content, String kid) {
+    public List<String> getChunkList(String content, DocumentSplitConfig config) {
-        return textSplitter.split(content, kid);
+        return textSplitter.split(content, config);
     }
 
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java

@@ -1,86 +1,17 @@
 package org.ruoyi.service.knowledge.impl.split;
 
-import lombok.AllArgsConstructor;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
-import lombok.extern.slf4j.Slf4j;
-import org.ruoyi.common.core.utils.StringUtils;
-import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
-import org.ruoyi.service.knowledge.IKnowledgeInfoService;
 import org.ruoyi.service.knowledge.TextSplitter;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Component;
 
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 @Component
-@Slf4j
 @Primary
-@AllArgsConstructor
 public class CharacterTextSplitter implements TextSplitter {
-
-    private final IKnowledgeInfoService knowledgeInfoService;
-
     @Override
-    public List<String> split(String content, String kid) {
+    public List<String> split(String content, DocumentSplitConfig config) {
-        // 默认配置值
+        return SplitterSupport.split(content, config, SplitterSupport::paragraphs);
-        String knowledgeSeparator = "#";
-        int textBlockSize = 1000;
-        int overlapChar = 50;
-
-        // 根据知识库ID查询配置，覆盖默认值
-        if (StringUtils.isNotBlank(kid)) {
-            try {
-                KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
-                if (info != null) {
-                    if (StringUtils.isNotBlank(info.getSeparator())) {
-                        knowledgeSeparator = info.getSeparator();
-                    }
-                    if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
-                        textBlockSize = info.getTextBlockSize().intValue();
-                    }
-                    if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
-                        overlapChar = info.getOverlapChar().intValue();
-                    }
-                }
-            } catch (Exception e) {
-                log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
-            }
-        }
-
-        List<String> chunkList = new ArrayList<>();
-        if (StringUtils.isNotBlank(knowledgeSeparator) && content.contains(knowledgeSeparator)) {
-            // 按自定义分隔符切分（字面量匹配，避免分隔符被当作正则）
-            String[] chunks = content.split(java.util.regex.Pattern.quote(knowledgeSeparator));
-            for (String chunk : chunks) {
-                if (StringUtils.isNotBlank(chunk)) {
-                    chunkList.add(chunk.trim());
-                }
-            }
-        } else {
-            int indexMin = 0;
-            int len = content.length();
-            int i = 0;
-            int right = 0;
-            while (true) {
-                if (len > right) {
-                    int begin = i * textBlockSize - overlapChar;
-                    if (begin < indexMin) {
-                        begin = indexMin;
-                    }
-                    int end = textBlockSize * (i + 1) + overlapChar;
-                    if (end > len) {
-                        end = len;
-                    }
-                    String chunk = content.substring(begin, end);
-                    chunkList.add(chunk);
-                    i++;
-                    right = right + textBlockSize;
-                } else {
-                    break;
-                }
-            }
-        }
-        return chunkList;
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java

@@ -1,47 +1,15 @@
 package org.ruoyi.service.knowledge.impl.split;
 
-import lombok.AllArgsConstructor;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
-import lombok.extern.slf4j.Slf4j;
-import org.ruoyi.common.core.utils.StringUtils;
-import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
-import org.ruoyi.service.knowledge.IKnowledgeInfoService;
 import org.ruoyi.service.knowledge.TextSplitter;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
 
-/**
- * 代码文件分片器：按空行（函数/类之间的自然边界）切分，
- * 块合并到不超过块大小，超大块再按滑动窗口切分
- */
 @Component
-@AllArgsConstructor
-@Slf4j
 public class CodeTextSplitter implements TextSplitter {
-
-    private final IKnowledgeInfoService knowledgeInfoService;
-
     @Override
-    public List<String> split(String content, String kid) {
+    public List<String> split(String content, DocumentSplitConfig config) {
-        int textBlockSize = 1000;
+        return SplitterSupport.split(content, config, SplitterSupport::paragraphs);
-        int overlapChar = 50;
-        if (StringUtils.isNotBlank(kid)) {
-            try {
-                KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
-                if (info != null) {
-                    if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
-                        textBlockSize = info.getTextBlockSize().intValue();
-                    }
-                    if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
-                        overlapChar = info.getOverlapChar().intValue();
-                    }
-                }
-            } catch (Exception e) {
-                log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
-            }
-        }
-        // 按空行切分，保留段落间的自然语义边界
-        String[] sections = content.split("\\n\\s*\\n");
-        return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar);
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/ExcelTextSplitter.java

@@ -1,79 +1,15 @@
 package org.ruoyi.service.knowledge.impl.split;
 
-import lombok.AllArgsConstructor;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
-import lombok.extern.slf4j.Slf4j;
-import org.ruoyi.common.core.utils.StringUtils;
-import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
-import org.ruoyi.service.knowledge.IKnowledgeInfoService;
 import org.ruoyi.service.knowledge.TextSplitter;
 import org.springframework.stereotype.Component;
 
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 @Component
-@AllArgsConstructor
-@Slf4j
 public class ExcelTextSplitter implements TextSplitter {
-
-    private final IKnowledgeInfoService knowledgeInfoService;
-
     @Override
-    public List<String> split(String content, String kid) {
+    public List<String> split(String content, DocumentSplitConfig config) {
-        // 默认配置
+        return SplitterSupport.split(content, config, SplitterSupport::lines);
-        String knowledgeSeparator = "#";
-        int textBlockSize = 1000;
-        int overlapChar = 50;
-
-        // 根据知识库ID查询配置，覆盖默认值
-        if (StringUtils.isNotBlank(kid)) {
-            try {
-                KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
-                if (info != null) {
-                    if (StringUtils.isNotBlank(info.getSeparator())) {
-                        knowledgeSeparator = info.getSeparator();
-                    }
-                    if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
-                        textBlockSize = info.getTextBlockSize().intValue();
-                    }
-                    if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
-                        overlapChar = info.getOverlapChar().intValue();
-                    }
-                }
-            } catch (Exception e) {
-                log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
-            }
-        }
-        List<String> chunkList = new ArrayList<>();
-        if (content.contains(knowledgeSeparator) && StringUtils.isNotBlank(knowledgeSeparator)) {
-            // 按自定义分隔符切分
-            String[] chunks = content.split(knowledgeSeparator);
-            chunkList.addAll(Arrays.asList(chunks));
-        } else {
-            int indexMin = 0;
-            int len = content.length();
-            int i = 0;
-            int right = 0;
-            while (true) {
-                if (len > right) {
-                    int begin = i * textBlockSize - overlapChar;
-                    if (begin < indexMin) {
-                        begin = indexMin;
-                    }
-                    int end = textBlockSize * (i + 1) + overlapChar;
-                    if (end > len) {
-                        end = len;
-                    }
-                    String chunk = content.substring(begin, end);
-                    chunkList.add(chunk);
-                    i++;
-                    right = right + textBlockSize;
-                } else {
-                    break;
-                }
-            }
-        }
-        return chunkList;
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java

@@ -1,47 +1,56 @@
 package org.ruoyi.service.knowledge.impl.split;
 
-import lombok.AllArgsConstructor;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
-import lombok.extern.slf4j.Slf4j;
-import org.ruoyi.common.core.utils.StringUtils;
-import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
-import org.ruoyi.service.knowledge.IKnowledgeInfoService;
 import org.ruoyi.service.knowledge.TextSplitter;
 import org.springframework.stereotype.Component;
 
+import java.util.ArrayList;
 import java.util.List;
 
-/**
- * Markdown 分片器：优先按标题（# ~ ######）切分，保持章节语义完整；
- * 小节合并到不超过块大小，超大章节再按滑动窗口切分
- */
 @Component
-@AllArgsConstructor
-@Slf4j
 public class MarkdownTextSplitter implements TextSplitter {
-
-    private final IKnowledgeInfoService knowledgeInfoService;
-
     @Override
-    public List<String> split(String content, String kid) {
+    public List<String> split(String content, DocumentSplitConfig config) {
-        int textBlockSize = 1000;
+        return SplitterSupport.split(content, config, this::sections);
-        int overlapChar = 50;
+    }
-        if (StringUtils.isNotBlank(kid)) {
+
-            try {
+    /** Split headings without treating heading-looking lines inside fenced code as headings. */
-                KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid));
+    private List<String> sections(String markdown) {
-                if (info != null) {
+        String[] lines = markdown.split("\\n", -1);
-                    if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) {
+        List<String> sections = new ArrayList<>();
-                        textBlockSize = info.getTextBlockSize().intValue();
+        StringBuilder current = new StringBuilder();
-                    }
+        boolean fenced = false;
-                    if (info.getOverlapChar() != null && info.getOverlapChar() > 0) {
+        String fenceMarker = null;
-                        overlapChar = info.getOverlapChar().intValue();
+        for (int i = 0; i < lines.length; i++) {
-                    }
+            String line = lines[i];
+            String trimmed = line.stripLeading();
+            if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) {
+                String marker = trimmed.substring(0, 3);
+                if (!fenced) {
+                    fenced = true;
+                    fenceMarker = marker;
+                } else if (marker.equals(fenceMarker)) {
+                    fenced = false;
+                    fenceMarker = null;
                 }
-            } catch (Exception e) {
+            }
-                log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e);
+            boolean atx = !fenced && line.matches("^#{1,6}(\\s+.*)?$");
+            boolean setextTitle = !fenced && i + 1 < lines.length
+                && lines[i + 1].matches("^\\s*(=+|-+)\\s*$") && !line.isBlank();
+            if ((atx || setextTitle) && current.length() > 0) flush(sections, current);
+            current.append(line);
+            if (i < lines.length - 1) current.append('\n');
+            if (setextTitle) {
+                current.append(lines[++i]);
+                if (i < lines.length - 1) current.append('\n');
             }
         }
-        // 按标题行切分（标题保留在各自小节开头）
+        flush(sections, current);
-        String[] sections = content.split("(?m)(?=^#{1,6}\\s)");
+        return sections;
-        return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar);
+    }
+
+    private static void flush(List<String> sections, StringBuilder current) {
+        if (!current.toString().isBlank()) sections.add(current.toString());
+        current.setLength(0);
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java

@@ -1,71 +1,67 @@
 package org.ruoyi.service.knowledge.impl.split;
 
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
+
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Function;
+import java.util.regex.Pattern;
 
-/**
+/** Shared, deterministic split pipeline used by every supported document format. */
- * 分片工具：提供各 Splitter 共用的滑动窗口切分与片段合并能力
- */
 public final class SplitterSupport {
+    private SplitterSupport() {}
 
-    private SplitterSupport() {
+    public static List<String> split(String content, DocumentSplitConfig config,
+                                     Function<String, List<String>> naturalSections) {
+        if (content == null || content.isBlank()) return List.of();
+        String normalized = content.replace("\r\n", "\n").replace('\r', '\n');
+        List<String> primary = literalSections(normalized, config.separator());
+        List<String> result = new ArrayList<>();
+        for (String part : primary) {
+            List<String> sections = naturalSections.apply(part);
+            String joined = sections.stream().filter(s -> s != null && !s.isBlank())
+                .map(String::strip).reduce((a, b) -> a + "\n\n" + b).orElse("");
+            result.addAll(slidingWindow(joined, config.blockSize(), config.overlap()));
+        }
+        return result;
     }
 
-    /**
+    static List<String> literalSections(String content, String separator) {
-     * 滑动窗口切分：每块约 blockSize 字符，相邻块保留 overlap 字符重叠
+        if (separator == null || separator.isEmpty() || !content.contains(separator)) return List.of(content);
-     */
+        List<String> parts = new ArrayList<>();
+        for (String part : content.split(Pattern.quote(separator), -1)) {
+            if (!part.isBlank()) parts.add(part);
+        }
+        return parts;
+    }
+
+    /** blockSize is a strict maximum; overlap is the repeated suffix/prefix length. */
     public static List<String> slidingWindow(String content, int blockSize, int overlap) {
-        List<String> chunkList = new ArrayList<>();
+        if (content == null || content.isBlank()) return List.of();
-        int len = content.length();
+        String value = content.strip();
-        int right = 0;
+        List<String> chunks = new ArrayList<>();
-        int i = 0;
+        int step = blockSize - overlap;
-        while (len > right) {
+        for (int start = 0; start < value.length(); start += step) {
-            int begin = i * blockSize - overlap;
+            int end = Math.min(value.length(), start + blockSize);
-            if (begin < 0) {
+            String chunk = value.substring(start, end).strip();
-                begin = 0;
+            if (!chunk.isEmpty()) chunks.add(chunk);
-            }
+            if (end == value.length()) break;
-            int end = blockSize * (i + 1) + overlap;
-            if (end > len) {
-                end = len;
-            }
-            String chunk = content.substring(begin, end).trim();
-            if (!chunk.isEmpty()) {
-                chunkList.add(chunk);
-            }
-            i++;
-            right = right + blockSize;
         }
-        return chunkList;
+        return chunks;
     }
 
-    /**
+    /** Compatibility helper retained for callers/tests; now enforces strict maximum size. */
-     * 将小段按顺序合并到不超过 blockSize，超过 blockSize 的单段再用滑动窗口切分
-     */
     public static List<String> mergeAndSplit(String[] sections, int blockSize, int overlap) {
-        List<String> chunkList = new ArrayList<>();
+        return split(String.join("\n\n", sections),
-        StringBuilder current = new StringBuilder();
+            new DocumentSplitConfig(null, blockSize, overlap, ""),
-        for (String section : sections) {
+            text -> List.of(text));
-            if (section == null || section.isBlank()) {
+    }
-                continue;
+
-            }
+    static List<String> paragraphs(String content) {
-            if (section.length() > blockSize) {
+        return List.of(content.split("\\n\\s*\\n+"));
-                // 超长段先冲刷当前缓冲，再单独窗口切分
+    }
-                if (current.length() > 0) {
+
-                    chunkList.add(current.toString().trim());
+    static List<String> lines(String content) {
-                    current.setLength(0);
+        return List.of(content.split("\\n+"));
-                }
-                chunkList.addAll(slidingWindow(section, blockSize, overlap));
-            } else if (current.length() + section.length() > blockSize) {
-                chunkList.add(current.toString().trim());
-                current.setLength(0);
-                current.append(section);
-            } else {
-                current.append(section);
-            }
-        }
-        if (current.length() > 0) {
-            chunkList.add(current.toString().trim());
-        }
-        return chunkList;
     }
 }

ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/TokenTextSplitter.java

@@ -3,6 +3,7 @@ package org.ruoyi.service.knowledge.impl.split;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.ruoyi.service.knowledge.TextSplitter;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
@@ -12,7 +13,7 @@ import java.util.List;
 @Slf4j
 public class TokenTextSplitter implements TextSplitter {
     @Override
-    public List<String> split(String content, String kid) {
+    public List<String> split(String content, DocumentSplitConfig config) {
-        return null;
+        return SplitterSupport.split(content, config, SplitterSupport::paragraphs);
     }
 }

ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java

@@ -1,78 +1,72 @@
 package org.ruoyi.service.knowledge.impl.split;
 
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Tag;
-import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo;
+import org.junit.jupiter.api.Test;
-import org.ruoyi.service.knowledge.IKnowledgeInfoService;
+import org.ruoyi.common.core.exception.ServiceException;
+import org.ruoyi.factory.ResourceLoaderFactory;
+import org.ruoyi.service.knowledge.DocumentSplitConfig;
+import org.ruoyi.service.knowledge.impl.loader.CodeFileLoader;
+import org.ruoyi.service.knowledge.impl.loader.MarkDownFileLoader;
 
 import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.*;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 @Tag("dev")
 class RagTextSplitterRegressionTest {
 
-    private static IKnowledgeInfoService knowledgeService(String separator, long blockSize, long overlap) {
+    @Test
-        IKnowledgeInfoService service = mock(IKnowledgeInfoService.class);
+    void allSplittersHonorLiteralSeparatorAndStrictMaximum() {
-        KnowledgeInfoVo info = new KnowledgeInfoVo();
+        List.of(new CharacterTextSplitter(), new MarkdownTextSplitter(),
-        info.setSeparator(separator);
+            new CodeTextSplitter(), new ExcelTextSplitter()).forEach(splitter -> {
-        info.setTextBlockSize(blockSize);
+            for (String separator : List.of("|", ".", "*", "<CUT>", "\n---\n")) {
-        info.setOverlapChar(overlap);
+                String content = "alpha" + separator + "b".repeat(35);
-        when(service.queryById(1L)).thenReturn(info);
+                List<String> chunks = splitter.split(content, config(separator, 12, 0));
-        return service;
+                assertTrue(chunks.size() >= 4, splitter.getClass().getSimpleName());
+                assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 12));
+                assertTrue(chunks.stream().noneMatch(chunk -> chunk.contains(separator)));
+            }
+        });
     }
 
     @Test
-    void characterSplitterTreatsRegexMetacharactersLiterally() {
+    void zeroOverlapIsRespectedAndConfiguredOverlapIsExact() {
-        CharacterTextSplitter pipe = new CharacterTextSplitter(knowledgeService("|", 1000, 50));
+        CharacterTextSplitter splitter = new CharacterTextSplitter();
-        assertEquals(List.of("alpha", "beta", "gamma"), pipe.split("alpha|beta|gamma", "1"));
+        assertEquals(List.of("abcdefghij", "klmnopqrst"),
-
+            splitter.split("abcdefghijklmnopqrst", config(null, 10, 0)));
-        CharacterTextSplitter dot = new CharacterTextSplitter(knowledgeService(".", 1000, 50));
+        List<String> overlap = splitter.split("abcdefghijklmnop", config(null, 10, 3));
-        assertEquals(List.of("alpha", "beta", "gamma"), dot.split("alpha.beta.gamma", "1"));
+        assertEquals("hij", overlap.get(0).substring(7));
-
+        assertTrue(overlap.get(1).startsWith("hij"));
-        CharacterTextSplitter star = new CharacterTextSplitter(knowledgeService("*", 1000, 50));
+        assertTrue(overlap.stream().allMatch(chunk -> chunk.length() <= 10));
-        assertEquals(List.of("alpha", "beta", "gamma"), star.split("alpha*beta*gamma", "1"));
     }
 
     @Test
-    void markdownSplitterReturnsNonEmptyBoundedChunks() {
+    void markdownKeepsHeadingsAndIgnoresHeadingsInsideFences() {
-        MarkdownTextSplitter splitter = new MarkdownTextSplitter(knowledgeService(null, 40, 5));
+        MarkdownTextSplitter splitter = new MarkdownTextSplitter();
-        String markdown = "# Title\nintro text\n## Details\n" + "detail ".repeat(20);
+        String markdown = "# Real\nintro\n```md\n# not a heading\n```\nTitle\n=====\nbody";
-
+        List<String> chunks = splitter.split(markdown, config(null, 200, 0));
-        List<String> chunks = splitter.split(markdown, "1");
+        assertEquals(1, chunks.size());
-
+        assertTrue(chunks.get(0).contains("# Real"));
-        assertFalse(chunks.isEmpty());
+        assertTrue(chunks.get(0).contains("# not a heading"));
-        assertTrue(chunks.stream().noneMatch(String::isBlank));
+        assertTrue(chunks.get(0).contains("Title\n====="));
-        assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 50),
-            "window size may include overlap on both sides");
-        assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("# Title")));
     }
 
     @Test
-    void codeSplitterReturnsNonEmptyChunksAndPreservesContent() {
+    void invalidConfigurationFailsClearly() {
-        CodeTextSplitter splitter = new CodeTextSplitter(knowledgeService(null, 45, 5));
+        assertThrows(ServiceException.class, () -> config(null, 0, 0));
-        String code = "class A {\n  void a() {}\n}\n\nclass B {\n" + "  int value = 1;\n".repeat(8) + "}";
+        assertThrows(ServiceException.class, () -> config(null, 10, 10));
-
+        assertThrows(ServiceException.class, () -> config(null, 10, -1));
-        List<String> chunks = splitter.split(code, "1");
-
-        assertFalse(chunks.isEmpty());
-        assertTrue(chunks.stream().noneMatch(String::isBlank));
-        assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class A")));
-        assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class B") || chunk.contains("int value")));
     }
 
     @Test
-    void splitterSupportHandlesEmptyAndOversizedSections() {
+    void loaderFactoryNormalizesSuffixAndUsesFormatSpecificLoader() {
-        assertTrue(SplitterSupport.mergeAndSplit(new String[]{"", "  "}, 20, 3).isEmpty());
+        ResourceLoaderFactory factory = new ResourceLoaderFactory(new CharacterTextSplitter(),
+            new CodeTextSplitter(), new MarkdownTextSplitter(), new ExcelTextSplitter());
+        assertInstanceOf(MarkDownFileLoader.class, factory.getLoaderByFileType(" .MD "));
+        assertInstanceOf(CodeFileLoader.class, factory.getLoaderByFileType(".JAVA"));
+    }
 
-        List<String> chunks = SplitterSupport.mergeAndSplit(
+    private DocumentSplitConfig config(String separator, int size, int overlap) {
-            new String[]{"short", "x".repeat(55)}, 20, 3);
+        return new DocumentSplitConfig(separator, size, overlap, "md");
-
-        assertEquals("short", chunks.get(0));
-        assertTrue(chunks.size() >= 4);
-        assertTrue(chunks.stream().noneMatch(String::isBlank));
-        assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 26));
     }
 }