diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/knowledge/KnowledgeAttachController.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/knowledge/KnowledgeAttachController.java index b5bab948..ea52cdf9 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/knowledge/KnowledgeAttachController.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/knowledge/KnowledgeAttachController.java @@ -9,6 +9,7 @@ import cn.dev33.satoken.annotation.SaCheckPermission; import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo; import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo; import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo; +import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo; import org.ruoyi.service.knowledge.IKnowledgeAttachService; import org.springframework.web.bind.annotation.*; import org.springframework.validation.annotation.Validated; @@ -129,4 +130,12 @@ public class KnowledgeAttachController extends BaseController { knowledgeAttachService.parse(id); return R.ok(); } + + @SaCheckPermission("system:attach:edit") + @Log(title = "知识库附件批量重新解析", businessType = BusinessType.UPDATE) + @PostMapping("/reparse/knowledge/{knowledgeId}") + @RepeatSubmit() + public R reparseKnowledge(@PathVariable Long knowledgeId) { + return R.ok(knowledgeAttachService.reparseKnowledge(knowledgeId)); + } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/knowledge/KnowledgeReparseVo.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/knowledge/KnowledgeReparseVo.java new file mode 100644 index 00000000..88f64bd2 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/knowledge/KnowledgeReparseVo.java @@ -0,0 +1,4 @@ +package org.ruoyi.domain.vo.knowledge; + +public record KnowledgeReparseVo(int submitted, int skipped, int total) { +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/ResourceLoaderFactory.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/ResourceLoaderFactory.java index de293426..0e5f601c 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/ResourceLoaderFactory.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/factory/ResourceLoaderFactory.java @@ -17,7 +17,7 @@ public class ResourceLoaderFactory { private final ExcelTextSplitter excelTextSplitter; public ResourceLoader getLoaderByFileType(String fileType) { - fileType = StringUtils.removeStart(fileType, "."); + fileType = StringUtils.lowerCase(StringUtils.removeStart(StringUtils.trim(fileType), ".")); if (FileTypeConstants.isTextFile(fileType)) { return new TextFileLoader(characterTextSplitter); } else if (FileTypeConstants.isWord(fileType)) { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/DocumentSplitConfig.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/DocumentSplitConfig.java new file mode 100644 index 00000000..c9511a57 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/DocumentSplitConfig.java @@ -0,0 +1,20 @@ +package org.ruoyi.service.knowledge; + +import org.ruoyi.common.core.exception.ServiceException; + +/** Immutable snapshot of the split settings used for one parse operation. */ +public record DocumentSplitConfig(String separator, int blockSize, int overlap, String fileType) { + + public static final int DEFAULT_BLOCK_SIZE = 1000; + public static final int DEFAULT_OVERLAP = 50; + + public DocumentSplitConfig { + if (blockSize <= 0) { + throw new ServiceException("文本块大小必须大于0"); + } + if (overlap < 0 || overlap >= blockSize) { + throw new ServiceException("重叠字符数必须大于等于0且小于文本块大小"); + } + fileType = fileType == null ? "" : fileType.strip().replaceFirst("^\\.", "").toLowerCase(); + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/IKnowledgeAttachService.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/IKnowledgeAttachService.java index 75d0a528..fb8ceb46 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/IKnowledgeAttachService.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/IKnowledgeAttachService.java @@ -5,6 +5,7 @@ import org.ruoyi.common.mybatis.core.page.PageQuery; import org.ruoyi.domain.bo.knowledge.KnowledgeAttachBo; import org.ruoyi.domain.bo.knowledge.KnowledgeInfoUploadBo; import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo; +import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo; import java.util.Collection; import java.util.List; @@ -79,4 +80,6 @@ public interface IKnowledgeAttachService { * @param id 附件ID */ void parse(Long id); + + KnowledgeReparseVo reparseKnowledge(Long knowledgeId); } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/ResourceLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/ResourceLoader.java index db17573f..e54888cc 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/ResourceLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/ResourceLoader.java @@ -10,5 +10,5 @@ public interface ResourceLoader { String getContent(InputStream inputStream); - List getChunkList(String content, String kid); + List getChunkList(String content, DocumentSplitConfig config); } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/TextSplitter.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/TextSplitter.java index 8dfab554..2e9510e6 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/TextSplitter.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/TextSplitter.java @@ -11,8 +11,8 @@ public interface TextSplitter { * 文本切分 * * @param content 文本内容 - * @param kid 知识库id + * @param config 本次解析的分片配置快照 * @return 切分后的文本列表 */ - List split(String content, String kid); + List split(String content, DocumentSplitConfig config); } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java index 1393d3fc..ed86abab 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeAttachServiceImpl.java @@ -27,12 +27,14 @@ import org.ruoyi.domain.entity.knowledge.KnowledgeFragment; import org.ruoyi.domain.vo.knowledge.DocFragmentCountVo; import org.ruoyi.domain.vo.knowledge.KnowledgeAttachVo; import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; +import org.ruoyi.domain.vo.knowledge.KnowledgeReparseVo; import org.ruoyi.factory.ResourceLoaderFactory; import org.ruoyi.mapper.knowledge.KnowledgeAttachMapper; import org.ruoyi.mapper.knowledge.KnowledgeFragmentMapper; import org.ruoyi.service.knowledge.IKnowledgeAttachService; import org.ruoyi.service.knowledge.IKnowledgeInfoService; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.vector.VectorStoreService; import org.ruoyi.service.retrieval.KnowledgeRetrievalService; import org.springframework.scheduling.annotation.Async; @@ -184,10 +186,17 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService { @Override public void parse(Long id) { KnowledgeAttach attach = baseMapper.selectById(id); - if (attach == null || (!KnowledgeAttachStatus.WAITING.getCode().equals(attach.getStatus()) && !KnowledgeAttachStatus.FAILED.getCode().equals(attach.getStatus()))) { + if (attach == null || KnowledgeAttachStatus.PARSING.getCode().equals(attach.getStatus())) { return; } + int claimed = baseMapper.update(null, Wrappers.lambdaUpdate() + .set(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode()) + .set(KnowledgeAttach::getRemark, null) + .eq(KnowledgeAttach::getId, id) + .ne(KnowledgeAttach::getStatus, KnowledgeAttachStatus.PARSING.getCode())); + if (claimed == 0) return; + try { attach.setStatus(KnowledgeAttachStatus.PARSING.getCode()); // 解析中 baseMapper.updateById(attach); @@ -196,6 +205,16 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService { Long knowledgeId = attach.getKnowledgeId(); String docId = attach.getDocId(); + KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId); + if (knowledgeInfoVo == null) { + throw new ServiceException("知识库不存在: " + knowledgeId); + } + int blockSize = knowledgeInfoVo.getTextBlockSize() == null + ? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : knowledgeInfoVo.getTextBlockSize().intValue(); + int overlap = knowledgeInfoVo.getOverlapChar() == null + ? DocumentSplitConfig.DEFAULT_OVERLAP : knowledgeInfoVo.getOverlapChar().intValue(); + DocumentSplitConfig splitConfig = new DocumentSplitConfig( + knowledgeInfoVo.getSeparator(), blockSize, overlap, attach.getType()); List oldFragments = knowledgeFragmentMapper.selectList( Wrappers.lambdaQuery().eq(KnowledgeFragment::getDocId, docId)); @@ -210,7 +229,7 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService { try (InputStream inputStream = new URL(ossDTO.getUrl()).openStream()) { content = resourceLoader.getContent(inputStream); } - List chunkList = resourceLoader.getChunkList(content, String.valueOf(knowledgeId)); + List chunkList = resourceLoader.getChunkList(content, splitConfig); if (CollUtil.isEmpty(chunkList)) { throw new RuntimeException("文档分片结果为空，请检查文档内容或分片器是否支持该文件类型"); @@ -231,7 +250,6 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService { knowledgeFragment.setCreateTime(new Date()); knowledgeFragmentList.add(knowledgeFragment); } - KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(knowledgeId); ChatModelVo chatModelVo = chatModelService.selectModelByName(knowledgeInfoVo.getEmbeddingModel()); StoreEmbeddingBo storeEmbeddingBo = new StoreEmbeddingBo(); @@ -275,4 +293,22 @@ public class KnowledgeAttachServiceImpl implements IKnowledgeAttachService { baseMapper.updateById(attach); } } + + @Override + public KnowledgeReparseVo reparseKnowledge(Long knowledgeId) { + List attachments = baseMapper.selectList( + Wrappers.lambdaQuery().eq(KnowledgeAttach::getKnowledgeId, knowledgeId)); + int submitted = 0; + int skipped = 0; + IKnowledgeAttachService proxy = SpringUtils.getBean(IKnowledgeAttachService.class); + for (KnowledgeAttach attachment : attachments) { + if (KnowledgeAttachStatus.PARSING.getCode().equals(attachment.getStatus())) { + skipped++; + } else { + proxy.parse(attachment.getId()); + submitted++; + } + } + return new KnowledgeReparseVo(submitted, skipped, attachments.size()); + } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeInfoServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeInfoServiceImpl.java index 2f96b97d..ce042dde 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeInfoServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/KnowledgeInfoServiceImpl.java @@ -18,6 +18,7 @@ import org.ruoyi.service.knowledge.IKnowledgeInfoService; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.ruoyi.service.retrieval.KnowledgeRetrievalService; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.common.core.service.OssService; import java.util.List; @@ -152,6 +153,11 @@ public class KnowledgeInfoServiceImpl implements IKnowledgeInfoService { * 保存前的数据校验 */ private void validEntityBeforeSave(KnowledgeInfo entity){ + int blockSize = entity.getTextBlockSize() == null + ? DocumentSplitConfig.DEFAULT_BLOCK_SIZE : entity.getTextBlockSize().intValue(); + int overlap = entity.getOverlapChar() == null + ? DocumentSplitConfig.DEFAULT_OVERLAP : entity.getOverlapChar().intValue(); + new DocumentSplitConfig(entity.getSeparator(), blockSize, overlap, ""); //TODO 做一些数据校验,如唯一约束 } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CodeFileLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CodeFileLoader.java index 11c5471d..942726f5 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CodeFileLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CodeFileLoader.java @@ -3,13 +3,16 @@ package org.ruoyi.service.knowledge.impl.loader; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; +import org.ruoyi.common.core.exception.ServiceException; import org.springframework.stereotype.Component; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.List; @Component @@ -21,20 +24,20 @@ public class CodeFileLoader implements ResourceLoader { @Override public String getContent(InputStream inputStream) { StringBuffer stringBuffer = new StringBuffer(); - try (InputStreamReader reader = new InputStreamReader(inputStream); + try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); BufferedReader bufferedReader = new BufferedReader(reader)) { String line; while ((line = bufferedReader.readLine()) != null) { stringBuffer.append(line).append("\n"); } } catch (IOException e) { - e.printStackTrace(); + throw new ServiceException("读取代码文件失败", e); } return stringBuffer.toString(); } @Override - public List getChunkList(String content, String kid) { - return textSplitter.split(content, kid); + public List getChunkList(String content, DocumentSplitConfig config) { + return textSplitter.split(content, config); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CsvFileLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CsvFileLoader.java index 12e2d668..b5174054 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CsvFileLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/CsvFileLoader.java @@ -1,6 +1,7 @@ package org.ruoyi.service.knowledge.impl.loader; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import java.io.InputStream; import java.util.List; @@ -12,7 +13,7 @@ public class CsvFileLoader implements ResourceLoader { } @Override - public List getChunkList(String content, String kid) { + public List getChunkList(String content, DocumentSplitConfig config) { return null; } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/ExcelFileLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/ExcelFileLoader.java index c13cee98..d9517453 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/ExcelFileLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/ExcelFileLoader.java @@ -6,6 +6,7 @@ import dev.langchain4j.data.document.parser.apache.tika.ApacheTikaDocumentParser import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; import org.springframework.stereotype.Component; @@ -38,7 +39,7 @@ public class ExcelFileLoader implements ResourceLoader { } @Override - public List getChunkList(String content, String kid) { - return textSplitter.split(content, kid); + public List getChunkList(String content, DocumentSplitConfig config) { + return textSplitter.split(content, config); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/FolderLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/FolderLoader.java index 3b344a1b..9456d9af 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/FolderLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/FolderLoader.java @@ -1,6 +1,7 @@ package org.ruoyi.service.knowledge.impl.loader; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import java.io.InputStream; import java.util.List; @@ -12,7 +13,7 @@ public class FolderLoader implements ResourceLoader { } @Override - public List getChunkList(String content, String kid) { + public List getChunkList(String content, DocumentSplitConfig config) { return null; } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/GithubLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/GithubLoader.java index 625d7feb..8fabc1b6 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/GithubLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/GithubLoader.java @@ -1,6 +1,7 @@ package org.ruoyi.service.knowledge.impl.loader; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import java.io.InputStream; import java.util.List; @@ -12,7 +13,7 @@ public class GithubLoader implements ResourceLoader { } @Override - public List getChunkList(String content, String kid) { + public List getChunkList(String content, DocumentSplitConfig config) { return null; } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/JsonFileLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/JsonFileLoader.java index 6f2b6041..2ee9e6d0 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/JsonFileLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/JsonFileLoader.java @@ -1,6 +1,7 @@ package org.ruoyi.service.knowledge.impl.loader; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import java.io.InputStream; import java.util.List; @@ -12,7 +13,7 @@ public class JsonFileLoader implements ResourceLoader { } @Override - public List getChunkList(String content, String kid) { + public List getChunkList(String content, DocumentSplitConfig config) { return null; } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/MarkDownFileLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/MarkDownFileLoader.java index 2137d991..105c6b88 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/MarkDownFileLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/MarkDownFileLoader.java @@ -3,13 +3,16 @@ package org.ruoyi.service.knowledge.impl.loader; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; +import org.ruoyi.common.core.exception.ServiceException; import org.springframework.stereotype.Component; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.List; @Component @@ -21,20 +24,20 @@ public class MarkDownFileLoader implements ResourceLoader { @Override public String getContent(InputStream inputStream) { StringBuffer stringBuffer = new StringBuffer(); - try (InputStreamReader reader = new InputStreamReader(inputStream); + try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); BufferedReader bufferedReader = new BufferedReader(reader)) { String line; while ((line = bufferedReader.readLine()) != null) { stringBuffer.append(line).append("\n"); } } catch (IOException e) { - e.printStackTrace(); + throw new ServiceException("读取 Markdown 文件失败", e); } return stringBuffer.toString(); } @Override - public List getChunkList(String content, String kid) { - return textSplitter.split(content, kid); + public List getChunkList(String content, DocumentSplitConfig config) { + return textSplitter.split(content, config); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/PdfFileLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/PdfFileLoader.java index 4fe3d5e4..9221dab5 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/PdfFileLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/PdfFileLoader.java @@ -6,6 +6,7 @@ import org.apache.pdfbox.io.RandomAccessReadBuffer; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.text.PDFTextStripper; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; import org.springframework.stereotype.Component; @@ -32,7 +33,7 @@ public class PdfFileLoader implements ResourceLoader { } @Override - public List getChunkList(String content, String kid) { - return characterTextSplitter.split(content, kid); + public List getChunkList(String content, DocumentSplitConfig config) { + return characterTextSplitter.split(content, config); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/TextFileLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/TextFileLoader.java index caf64d95..9478f0ab 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/TextFileLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/TextFileLoader.java @@ -3,7 +3,9 @@ package org.ruoyi.service.knowledge.impl.loader; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; +import org.ruoyi.common.core.exception.ServiceException; import org.springframework.stereotype.Component; import java.io.BufferedReader; @@ -22,18 +24,16 @@ public class TextFileLoader implements ResourceLoader { @Override public String getContent(InputStream inputStream) { - String stringBuffer = ""; try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); BufferedReader bufferedReader = new BufferedReader(reader)) { - stringBuffer = bufferedReader.lines().collect(Collectors.joining()); + return bufferedReader.lines().collect(Collectors.joining("\n")); } catch (IOException e) { - e.printStackTrace(); + throw new ServiceException("读取文本文件失败", e); } - return stringBuffer; } @Override - public List getChunkList(String content, String kid) { - return textSplitter.split(content, kid); + public List getChunkList(String content, DocumentSplitConfig config) { + return textSplitter.split(content, config); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/WordLoader.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/WordLoader.java index bede5a7e..3098be99 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/WordLoader.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/loader/WordLoader.java @@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.poi.xwpf.extractor.XWPFWordExtractor; import org.apache.poi.xwpf.usermodel.XWPFDocument; import org.ruoyi.service.knowledge.ResourceLoader; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; import org.springframework.stereotype.Component; @@ -32,8 +33,8 @@ public class WordLoader implements ResourceLoader { } @Override - public List getChunkList(String content, String kid) { - return textSplitter.split(content, kid); + public List getChunkList(String content, DocumentSplitConfig config) { + return textSplitter.split(content, config); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java index 950608f6..5d958ce8 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CharacterTextSplitter.java @@ -1,86 +1,17 @@ package org.ruoyi.service.knowledge.impl.split; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.ruoyi.common.core.utils.StringUtils; -import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; -import org.ruoyi.service.knowledge.IKnowledgeInfoService; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; @Component -@Slf4j @Primary -@AllArgsConstructor public class CharacterTextSplitter implements TextSplitter { - - private final IKnowledgeInfoService knowledgeInfoService; - @Override - public List split(String content, String kid) { - // 默认配置值 - String knowledgeSeparator = "#"; - int textBlockSize = 1000; - int overlapChar = 50; - - // 根据知识库ID查询配置，覆盖默认值 - if (StringUtils.isNotBlank(kid)) { - try { - KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid)); - if (info != null) { - if (StringUtils.isNotBlank(info.getSeparator())) { - knowledgeSeparator = info.getSeparator(); - } - if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) { - textBlockSize = info.getTextBlockSize().intValue(); - } - if (info.getOverlapChar() != null && info.getOverlapChar() > 0) { - overlapChar = info.getOverlapChar().intValue(); - } - } - } catch (Exception e) { - log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e); - } - } - - List chunkList = new ArrayList<>(); - if (StringUtils.isNotBlank(knowledgeSeparator) && content.contains(knowledgeSeparator)) { - // 按自定义分隔符切分（字面量匹配，避免分隔符被当作正则） - String[] chunks = content.split(java.util.regex.Pattern.quote(knowledgeSeparator)); - for (String chunk : chunks) { - if (StringUtils.isNotBlank(chunk)) { - chunkList.add(chunk.trim()); - } - } - } else { - int indexMin = 0; - int len = content.length(); - int i = 0; - int right = 0; - while (true) { - if (len > right) { - int begin = i * textBlockSize - overlapChar; - if (begin < indexMin) { - begin = indexMin; - } - int end = textBlockSize * (i + 1) + overlapChar; - if (end > len) { - end = len; - } - String chunk = content.substring(begin, end); - chunkList.add(chunk); - i++; - right = right + textBlockSize; - } else { - break; - } - } - } - return chunkList; + public List split(String content, DocumentSplitConfig config) { + return SplitterSupport.split(content, config, SplitterSupport::paragraphs); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java index 98badf0b..34288308 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/CodeTextSplitter.java @@ -1,47 +1,15 @@ package org.ruoyi.service.knowledge.impl.split; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.ruoyi.common.core.utils.StringUtils; -import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; -import org.ruoyi.service.knowledge.IKnowledgeInfoService; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; import org.springframework.stereotype.Component; import java.util.List; -/** - * 代码文件分片器：按空行（函数/类之间的自然边界）切分， - * 块合并到不超过块大小，超大块再按滑动窗口切分 - */ @Component -@AllArgsConstructor -@Slf4j public class CodeTextSplitter implements TextSplitter { - - private final IKnowledgeInfoService knowledgeInfoService; - @Override - public List split(String content, String kid) { - int textBlockSize = 1000; - int overlapChar = 50; - if (StringUtils.isNotBlank(kid)) { - try { - KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid)); - if (info != null) { - if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) { - textBlockSize = info.getTextBlockSize().intValue(); - } - if (info.getOverlapChar() != null && info.getOverlapChar() > 0) { - overlapChar = info.getOverlapChar().intValue(); - } - } - } catch (Exception e) { - log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e); - } - } - // 按空行切分，保留段落间的自然语义边界 - String[] sections = content.split("\\n\\s*\\n"); - return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar); + public List split(String content, DocumentSplitConfig config) { + return SplitterSupport.split(content, config, SplitterSupport::paragraphs); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/ExcelTextSplitter.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/ExcelTextSplitter.java index 1b610107..3ac57c4e 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/ExcelTextSplitter.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/ExcelTextSplitter.java @@ -1,79 +1,15 @@ package org.ruoyi.service.knowledge.impl.split; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.ruoyi.common.core.utils.StringUtils; -import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; -import org.ruoyi.service.knowledge.IKnowledgeInfoService; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; @Component -@AllArgsConstructor -@Slf4j public class ExcelTextSplitter implements TextSplitter { - - private final IKnowledgeInfoService knowledgeInfoService; - @Override - public List split(String content, String kid) { - // 默认配置 - String knowledgeSeparator = "#"; - int textBlockSize = 1000; - int overlapChar = 50; - - // 根据知识库ID查询配置，覆盖默认值 - if (StringUtils.isNotBlank(kid)) { - try { - KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid)); - if (info != null) { - if (StringUtils.isNotBlank(info.getSeparator())) { - knowledgeSeparator = info.getSeparator(); - } - if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) { - textBlockSize = info.getTextBlockSize().intValue(); - } - if (info.getOverlapChar() != null && info.getOverlapChar() > 0) { - overlapChar = info.getOverlapChar().intValue(); - } - } - } catch (Exception e) { - log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e); - } - } - List chunkList = new ArrayList<>(); - if (content.contains(knowledgeSeparator) && StringUtils.isNotBlank(knowledgeSeparator)) { - // 按自定义分隔符切分 - String[] chunks = content.split(knowledgeSeparator); - chunkList.addAll(Arrays.asList(chunks)); - } else { - int indexMin = 0; - int len = content.length(); - int i = 0; - int right = 0; - while (true) { - if (len > right) { - int begin = i * textBlockSize - overlapChar; - if (begin < indexMin) { - begin = indexMin; - } - int end = textBlockSize * (i + 1) + overlapChar; - if (end > len) { - end = len; - } - String chunk = content.substring(begin, end); - chunkList.add(chunk); - i++; - right = right + textBlockSize; - } else { - break; - } - } - } - return chunkList; + public List split(String content, DocumentSplitConfig config) { + return SplitterSupport.split(content, config, SplitterSupport::lines); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java index 38358749..593d61cd 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/MarkdownTextSplitter.java @@ -1,47 +1,56 @@ package org.ruoyi.service.knowledge.impl.split; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.ruoyi.common.core.utils.StringUtils; -import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; -import org.ruoyi.service.knowledge.IKnowledgeInfoService; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.ruoyi.service.knowledge.TextSplitter; import org.springframework.stereotype.Component; +import java.util.ArrayList; import java.util.List; -/** - * Markdown 分片器：优先按标题（# ~ ######）切分，保持章节语义完整； - * 小节合并到不超过块大小，超大章节再按滑动窗口切分 - */ @Component -@AllArgsConstructor -@Slf4j public class MarkdownTextSplitter implements TextSplitter { - - private final IKnowledgeInfoService knowledgeInfoService; - @Override - public List split(String content, String kid) { - int textBlockSize = 1000; - int overlapChar = 50; - if (StringUtils.isNotBlank(kid)) { - try { - KnowledgeInfoVo info = knowledgeInfoService.queryById(Long.parseLong(kid)); - if (info != null) { - if (info.getTextBlockSize() != null && info.getTextBlockSize() > 0) { - textBlockSize = info.getTextBlockSize().intValue(); - } - if (info.getOverlapChar() != null && info.getOverlapChar() > 0) { - overlapChar = info.getOverlapChar().intValue(); - } + public List split(String content, DocumentSplitConfig config) { + return SplitterSupport.split(content, config, this::sections); + } + + /** Split headings without treating heading-looking lines inside fenced code as headings. */ + private List sections(String markdown) { + String[] lines = markdown.split("\\n", -1); + List sections = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + boolean fenced = false; + String fenceMarker = null; + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + String trimmed = line.stripLeading(); + if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) { + String marker = trimmed.substring(0, 3); + if (!fenced) { + fenced = true; + fenceMarker = marker; + } else if (marker.equals(fenceMarker)) { + fenced = false; + fenceMarker = null; } - } catch (Exception e) { - log.warn("查询知识库配置失败，使用默认配置, kid={}", kid, e); + } + boolean atx = !fenced && line.matches("^#{1,6}(\\s+.*)?$"); + boolean setextTitle = !fenced && i + 1 < lines.length + && lines[i + 1].matches("^\\s*(=+|-+)\\s*$") && !line.isBlank(); + if ((atx || setextTitle) && current.length() > 0) flush(sections, current); + current.append(line); + if (i < lines.length - 1) current.append('\n'); + if (setextTitle) { + current.append(lines[++i]); + if (i < lines.length - 1) current.append('\n'); } } - // 按标题行切分（标题保留在各自小节开头） - String[] sections = content.split("(?m)(?=^#{1,6}\\s)"); - return SplitterSupport.mergeAndSplit(sections, textBlockSize, overlapChar); + flush(sections, current); + return sections; + } + + private static void flush(List sections, StringBuilder current) { + if (!current.toString().isBlank()) sections.add(current.toString()); + current.setLength(0); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java index dbf91360..59ea7820 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/SplitterSupport.java @@ -1,71 +1,67 @@ package org.ruoyi.service.knowledge.impl.split; +import org.ruoyi.service.knowledge.DocumentSplitConfig; + import java.util.ArrayList; import java.util.List; +import java.util.function.Function; +import java.util.regex.Pattern; -/** - * 分片工具：提供各 Splitter 共用的滑动窗口切分与片段合并能力 - */ +/** Shared, deterministic split pipeline used by every supported document format. */ public final class SplitterSupport { + private SplitterSupport() {} - private SplitterSupport() { + public static List split(String content, DocumentSplitConfig config, + Function> naturalSections) { + if (content == null || content.isBlank()) return List.of(); + String normalized = content.replace("\r\n", "\n").replace('\r', '\n'); + List primary = literalSections(normalized, config.separator()); + List result = new ArrayList<>(); + for (String part : primary) { + List sections = naturalSections.apply(part); + String joined = sections.stream().filter(s -> s != null && !s.isBlank()) + .map(String::strip).reduce((a, b) -> a + "\n\n" + b).orElse(""); + result.addAll(slidingWindow(joined, config.blockSize(), config.overlap())); + } + return result; } - /** - * 滑动窗口切分：每块约 blockSize 字符，相邻块保留 overlap 字符重叠 - */ + static List literalSections(String content, String separator) { + if (separator == null || separator.isEmpty() || !content.contains(separator)) return List.of(content); + List parts = new ArrayList<>(); + for (String part : content.split(Pattern.quote(separator), -1)) { + if (!part.isBlank()) parts.add(part); + } + return parts; + } + + /** blockSize is a strict maximum; overlap is the repeated suffix/prefix length. */ public static List slidingWindow(String content, int blockSize, int overlap) { - List chunkList = new ArrayList<>(); - int len = content.length(); - int right = 0; - int i = 0; - while (len > right) { - int begin = i * blockSize - overlap; - if (begin < 0) { - begin = 0; - } - int end = blockSize * (i + 1) + overlap; - if (end > len) { - end = len; - } - String chunk = content.substring(begin, end).trim(); - if (!chunk.isEmpty()) { - chunkList.add(chunk); - } - i++; - right = right + blockSize; + if (content == null || content.isBlank()) return List.of(); + String value = content.strip(); + List chunks = new ArrayList<>(); + int step = blockSize - overlap; + for (int start = 0; start < value.length(); start += step) { + int end = Math.min(value.length(), start + blockSize); + String chunk = value.substring(start, end).strip(); + if (!chunk.isEmpty()) chunks.add(chunk); + if (end == value.length()) break; } - return chunkList; + return chunks; } - /** - * 将小段按顺序合并到不超过 blockSize，超过 blockSize 的单段再用滑动窗口切分 - */ + /** Compatibility helper retained for callers/tests; now enforces strict maximum size. */ public static List mergeAndSplit(String[] sections, int blockSize, int overlap) { - List chunkList = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - for (String section : sections) { - if (section == null || section.isBlank()) { - continue; - } - if (section.length() > blockSize) { - // 超长段先冲刷当前缓冲，再单独窗口切分 - if (current.length() > 0) { - chunkList.add(current.toString().trim()); - current.setLength(0); - } - chunkList.addAll(slidingWindow(section, blockSize, overlap)); - } else if (current.length() + section.length() > blockSize) { - chunkList.add(current.toString().trim()); - current.setLength(0); - current.append(section); - } else { - current.append(section); - } - } - if (current.length() > 0) { - chunkList.add(current.toString().trim()); - } - return chunkList; + return split(String.join("\n\n", sections), + new DocumentSplitConfig(null, blockSize, overlap, ""), + text -> List.of(text)); + } + + static List paragraphs(String content) { + return List.of(content.split("\\n\\s*\\n+")); + } + + static List lines(String content) { + return List.of(content.split("\\n+")); } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/TokenTextSplitter.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/TokenTextSplitter.java index 841adda2..34d577a1 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/TokenTextSplitter.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/knowledge/impl/split/TokenTextSplitter.java @@ -3,6 +3,7 @@ package org.ruoyi.service.knowledge.impl.split; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.ruoyi.service.knowledge.TextSplitter; +import org.ruoyi.service.knowledge.DocumentSplitConfig; import org.springframework.stereotype.Component; import java.util.List; @@ -12,7 +13,7 @@ import java.util.List; @Slf4j public class TokenTextSplitter implements TextSplitter { @Override - public List split(String content, String kid) { - return null; + public List split(String content, DocumentSplitConfig config) { + return SplitterSupport.split(content, config, SplitterSupport::paragraphs); } } diff --git a/ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java b/ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java index b3f2e151..bcc216e6 100644 --- a/ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java +++ b/ruoyi-modules/ruoyi-chat/src/test/java/org/ruoyi/service/knowledge/impl/split/RagTextSplitterRegressionTest.java @@ -1,78 +1,72 @@ package org.ruoyi.service.knowledge.impl.split; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Tag; -import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; -import org.ruoyi.service.knowledge.IKnowledgeInfoService; +import org.junit.jupiter.api.Test; +import org.ruoyi.common.core.exception.ServiceException; +import org.ruoyi.factory.ResourceLoaderFactory; +import org.ruoyi.service.knowledge.DocumentSplitConfig; +import org.ruoyi.service.knowledge.impl.loader.CodeFileLoader; +import org.ruoyi.service.knowledge.impl.loader.MarkDownFileLoader; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; @Tag("dev") class RagTextSplitterRegressionTest { - private static IKnowledgeInfoService knowledgeService(String separator, long blockSize, long overlap) { - IKnowledgeInfoService service = mock(IKnowledgeInfoService.class); - KnowledgeInfoVo info = new KnowledgeInfoVo(); - info.setSeparator(separator); - info.setTextBlockSize(blockSize); - info.setOverlapChar(overlap); - when(service.queryById(1L)).thenReturn(info); - return service; + @Test + void allSplittersHonorLiteralSeparatorAndStrictMaximum() { + List.of(new CharacterTextSplitter(), new MarkdownTextSplitter(), + new CodeTextSplitter(), new ExcelTextSplitter()).forEach(splitter -> { + for (String separator : List.of("|", ".", "*", "", "\n---\n")) { + String content = "alpha" + separator + "b".repeat(35); + List chunks = splitter.split(content, config(separator, 12, 0)); + assertTrue(chunks.size() >= 4, splitter.getClass().getSimpleName()); + assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 12)); + assertTrue(chunks.stream().noneMatch(chunk -> chunk.contains(separator))); + } + }); } @Test - void characterSplitterTreatsRegexMetacharactersLiterally() { - CharacterTextSplitter pipe = new CharacterTextSplitter(knowledgeService("|", 1000, 50)); - assertEquals(List.of("alpha", "beta", "gamma"), pipe.split("alpha|beta|gamma", "1")); - - CharacterTextSplitter dot = new CharacterTextSplitter(knowledgeService(".", 1000, 50)); - assertEquals(List.of("alpha", "beta", "gamma"), dot.split("alpha.beta.gamma", "1")); - - CharacterTextSplitter star = new CharacterTextSplitter(knowledgeService("*", 1000, 50)); - assertEquals(List.of("alpha", "beta", "gamma"), star.split("alpha*beta*gamma", "1")); + void zeroOverlapIsRespectedAndConfiguredOverlapIsExact() { + CharacterTextSplitter splitter = new CharacterTextSplitter(); + assertEquals(List.of("abcdefghij", "klmnopqrst"), + splitter.split("abcdefghijklmnopqrst", config(null, 10, 0))); + List overlap = splitter.split("abcdefghijklmnop", config(null, 10, 3)); + assertEquals("hij", overlap.get(0).substring(7)); + assertTrue(overlap.get(1).startsWith("hij")); + assertTrue(overlap.stream().allMatch(chunk -> chunk.length() <= 10)); } @Test - void markdownSplitterReturnsNonEmptyBoundedChunks() { - MarkdownTextSplitter splitter = new MarkdownTextSplitter(knowledgeService(null, 40, 5)); - String markdown = "# Title\nintro text\n## Details\n" + "detail ".repeat(20); - - List chunks = splitter.split(markdown, "1"); - - assertFalse(chunks.isEmpty()); - assertTrue(chunks.stream().noneMatch(String::isBlank)); - assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 50), - "window size may include overlap on both sides"); - assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("# Title"))); + void markdownKeepsHeadingsAndIgnoresHeadingsInsideFences() { + MarkdownTextSplitter splitter = new MarkdownTextSplitter(); + String markdown = "# Real\nintro\n```md\n# not a heading\n```\nTitle\n=====\nbody"; + List chunks = splitter.split(markdown, config(null, 200, 0)); + assertEquals(1, chunks.size()); + assertTrue(chunks.get(0).contains("# Real")); + assertTrue(chunks.get(0).contains("# not a heading")); + assertTrue(chunks.get(0).contains("Title\n=====")); } @Test - void codeSplitterReturnsNonEmptyChunksAndPreservesContent() { - CodeTextSplitter splitter = new CodeTextSplitter(knowledgeService(null, 45, 5)); - String code = "class A {\n void a() {}\n}\n\nclass B {\n" + " int value = 1;\n".repeat(8) + "}"; - - List chunks = splitter.split(code, "1"); - - assertFalse(chunks.isEmpty()); - assertTrue(chunks.stream().noneMatch(String::isBlank)); - assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class A"))); - assertTrue(chunks.stream().anyMatch(chunk -> chunk.contains("class B") || chunk.contains("int value"))); + void invalidConfigurationFailsClearly() { + assertThrows(ServiceException.class, () -> config(null, 0, 0)); + assertThrows(ServiceException.class, () -> config(null, 10, 10)); + assertThrows(ServiceException.class, () -> config(null, 10, -1)); } @Test - void splitterSupportHandlesEmptyAndOversizedSections() { - assertTrue(SplitterSupport.mergeAndSplit(new String[]{"", " "}, 20, 3).isEmpty()); + void loaderFactoryNormalizesSuffixAndUsesFormatSpecificLoader() { + ResourceLoaderFactory factory = new ResourceLoaderFactory(new CharacterTextSplitter(), + new CodeTextSplitter(), new MarkdownTextSplitter(), new ExcelTextSplitter()); + assertInstanceOf(MarkDownFileLoader.class, factory.getLoaderByFileType(" .MD ")); + assertInstanceOf(CodeFileLoader.class, factory.getLoaderByFileType(".JAVA")); + } - List chunks = SplitterSupport.mergeAndSplit( - new String[]{"short", "x".repeat(55)}, 20, 3); - - assertEquals("short", chunks.get(0)); - assertTrue(chunks.size() >= 4); - assertTrue(chunks.stream().noneMatch(String::isBlank)); - assertTrue(chunks.stream().allMatch(chunk -> chunk.length() <= 26)); + private DocumentSplitConfig config(String separator, int size, int overlap) { + return new DocumentSplitConfig(separator, size, overlap, "md"); } }