xieyinlu/ruoyi-ai
1
0
Fork 0
mirror of https://gitcode.com/ageerle/ruoyi-ai.git synced 2026-04-13 03:45:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: 调整知识库模块

Browse Source
This commit is contained in:
ageerle
2025-04-09 17:41:29 +08:00
parent be6d027cad
commit 3be9005f95
424 changed files with 1584 additions and 10005 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ruoyi-modules/pom.xml
View File

@@ -20,7 +20,6 @@
<module>ruoyi-demo</module>
<module>ruoyi-chat</module>
<module>ruoyi-system</module>
<module>ruoyi-knowledge</module>
<module>ruoyi-generator</module>
</modules>

16
ruoyi-modules/ruoyi-chat/pom.xml
View File

@@ -107,13 +107,27 @@
<groupId>jakarta.servlet</groupId>
<artifactId>jakarta.servlet-api</artifactId>
</dependency>
<dependency>
<groupId>org.ruoyi</groupId>
<artifactId>ruoyi-system</artifactId>
</dependency>
<dependency>
<groupId>org.ruoyi</groupId>
<artifactId>ruoyi-knowledge</artifactId>
<artifactId>ruoyi-knowledge-api</artifactId>
</dependency>
<dependency>
<groupId>org.ruoyi</groupId>
<artifactId>ruoyi-chat-api</artifactId>
<version>1.0.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.26</version>
<scope>compile</scope>
</dependency>
</dependencies>

29
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/controller/ChatController.java
View File

@@ -5,11 +5,12 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.chat.service.chat.ISseService;
import org.ruoyi.common.chat.domain.request.ChatRequest;
import org.ruoyi.common.chat.domain.request.Dall3Request;
import org.ruoyi.common.chat.entity.Tts.TextToSpeech;
import org.ruoyi.common.chat.entity.files.UploadFileResponse;
import org.ruoyi.common.chat.entity.images.Item;
import org.ruoyi.common.chat.entity.whisper.WhisperResponse;
import org.ruoyi.common.core.domain.R;
import org.ruoyi.common.core.domain.model.LoginUser;
@@ -17,8 +18,10 @@ import org.ruoyi.common.core.exception.base.BaseException;
import org.ruoyi.common.mybatis.core.page.PageQuery;
import org.ruoyi.common.mybatis.core.page.TableDataInfo;
import org.ruoyi.common.satoken.utils.LoginHelper;
import org.ruoyi.system.domain.request.translation.TranslationRequest;
import org.ruoyi.system.service.ISseService;
import org.ruoyi.domain.bo.ChatMessageBo;
import org.ruoyi.domain.vo.ChatMessageVo;
import org.ruoyi.service.IChatMessageService;
import org.springframework.core.io.Resource;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
@@ -26,7 +29,6 @@ import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.util.List;
/**
* 描述：聊天管理
@@ -56,7 +58,6 @@ public class ChatController {
return sseService.sseChat(chatRequest,request);
}
/**
* 上传文件
*/
@@ -90,22 +91,6 @@ public class ChatController {
return sseService.textToSpeed(textToSpeech);
}
/**
* 文本翻译
*
* @param
*/
@PostMapping("/translation")
@ResponseBody
public String translation(@RequestBody TranslationRequest translationRequest) {
return sseService.translation(translationRequest);
}
@PostMapping("/dall3")
@ResponseBody
public R<List<Item>> dall3(@RequestBody @Valid Dall3Request request) {
return R.ok(sseService.dall3(request));
}
/**
* 聊天记录

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/controller/SubmitController.java
View File

@@ -7,7 +7,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import okhttp3.Request;
import org.apache.commons.lang3.math.NumberUtils;
import org.ruoyi.chat.dto.*;
import org.ruoyi.chat.domain.dto.*;
import org.ruoyi.chat.enums.ActionType;
import org.ruoyi.chat.util.MjOkHttpUtil;
import org.springframework.web.bind.annotation.PostMapping;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/controller/TaskController.java
View File

@@ -7,7 +7,7 @@ import io.swagger.annotations.ApiParam;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import okhttp3.Request;
import org.ruoyi.chat.dto.TaskConditionDTO;
import org.ruoyi.chat.domain.dto.TaskConditionDTO;
import org.ruoyi.chat.util.MjOkHttpUtil;
import org.springframework.web.bind.annotation.*;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/BaseSubmitDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/BaseSubmitDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Getter;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/SubmitActionDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/SubmitActionDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModel;
import lombok.Data;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/SubmitBlendDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/SubmitBlendDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/SubmitChangeDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/SubmitChangeDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/SubmitDescribeDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/SubmitDescribeDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/SubmitImagineDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/SubmitImagineDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/SubmitModalDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/SubmitModalDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModel;
import lombok.Data;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/SubmitShortenDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/SubmitShortenDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModel;
import lombok.Data;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/SubmitSimpleChangeDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/SubmitSimpleChangeDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;

2
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/dto/TaskConditionDTO.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/domain/dto/TaskConditionDTO.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.chat.dto;
package org.ruoyi.chat.domain.dto;
import io.swagger.annotations.ApiModel;
import lombok.Data;

53
ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/listener/SSEEventSourceListener.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/listener/SSEEventSourceListener.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.system.listener;
package org.ruoyi.chat.listener;
import cn.hutool.core.collection.CollectionUtil;
@@ -10,20 +10,13 @@ import okhttp3.Response;
import okhttp3.ResponseBody;
import okhttp3.sse.EventSource;
import okhttp3.sse.EventSourceListener;
import org.ruoyi.common.chat.config.LocalCache;
import org.ruoyi.common.chat.entity.chat.ChatCompletionResponse;
import org.ruoyi.common.chat.utils.TikTokensUtil;
import org.ruoyi.common.core.utils.SpringUtils;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.system.domain.bo.SysModelBo;
import org.ruoyi.system.domain.vo.SysModelVo;
import org.ruoyi.system.service.ISysModelService;
import org.jetbrains.annotations.NotNull;
import org.ruoyi.common.chat.entity.chat.ChatCompletionResponse;
import org.ruoyi.common.core.utils.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter;
import java.util.List;
import java.util.Objects;
/**
@@ -37,15 +30,15 @@ import java.util.Objects;
@Component
public class SSEEventSourceListener extends EventSourceListener {
private ResponseBodyEmitter emitter;
private StringBuilder stringBuffer = new StringBuilder();
@Autowired(required = false)
public SSEEventSourceListener(ResponseBodyEmitter emitter) {
this.emitter = emitter;
}
private static final ISysModelService sysModelService = SpringUtils.getBean(ISysModelService.class);
private ResponseBodyEmitter emitter;
private StringBuilder stringBuffer;
private String modelName;
/**
* {@inheritDoc}
@@ -65,34 +58,8 @@ public class SSEEventSourceListener extends EventSourceListener {
if ("[DONE]".equals(data)) {
//成功响应
emitter.complete();
if(StringUtils.isNotEmpty(modelName)){
IChatCostService IChatCostService = SpringUtils.context().getBean(IChatCostService.class);
IChatMessageService chatMessageService = SpringUtils.context().getBean(IChatMessageService.class);
ChatMessageBo chatMessageBo = new ChatMessageBo();
chatMessageBo.setModelName(modelName);
chatMessageBo.setContent(stringBuffer.toString());
Long userId = (Long)LocalCache.CACHE.get("userId");
if(userId == null){
return;
}
chatMessageBo.setUserId(userId);
//查询按次数扣费的模型
SysModelBo sysModelBo = new SysModelBo();
sysModelBo.setModelType("2");
sysModelBo.setModelName(modelName);
List<SysModelVo> sysModelList = sysModelService.queryList(sysModelBo);
if (CollectionUtil.isNotEmpty(sysModelList)){
chatMessageBo.setDeductCost(0d);
chatMessageBo.setRemark("提问时扣费");
// 保存消息记录
chatMessageService.insertByBo(chatMessageBo);
}else{
int tokens = TikTokensUtil.tokens(modelName,stringBuffer.toString());
chatMessageBo.setTotalTokens(tokens);
// 按token扣费并且保存消息记录
IChatCostService.deductToken(chatMessageBo);
}
}
// 扣除费用 (消耗字符 模型名称)
return;
}
// 解析返回内容

43
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/IChatCostService.java Normal file
View File

@@ -0,0 +1,43 @@
package org.ruoyi.chat.service.chat;
import org.ruoyi.domain.bo.ChatMessageBo;
/**
* 计费管理Service接口
*
* @author ageerle
* @date 2025-04-08
*/
public interface IChatCostService {
/**
* 根据消耗的tokens扣除余额
*
* @param chatMessageBo
* @return 结果
*/
void deductToken(ChatMessageBo chatMessageBo);
/**
* 扣除用户的余额
*
*/
void deductUserBalance(Long userId, Double numberCost);
/**
* 扣除任务费用并且保存记录
*
* @param type 任务类型
* @param prompt 任务描述
* @param cost 扣除费用
*/
void taskDeduct(String type,String prompt, double cost);
/**
* 判断用户是否付费
*/
void checkUserGrade();
}

51
ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/ISseService.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/ISseService.java
View File

@@ -1,75 +1,58 @@
package org.ruoyi.system.service;
package org.ruoyi.chat.service.chat;
import jakarta.servlet.http.HttpServletRequest;
import org.ruoyi.common.chat.domain.request.ChatRequest;
import org.ruoyi.common.chat.domain.request.Dall3Request;
import org.ruoyi.common.chat.entity.Tts.TextToSpeech;
import org.ruoyi.common.chat.entity.files.UploadFileResponse;
import org.ruoyi.common.chat.entity.images.Item;
import org.ruoyi.common.chat.entity.whisper.WhisperResponse;
import org.ruoyi.system.domain.request.translation.TranslationRequest;
import org.springframework.core.io.Resource;
import org.springframework.http.ResponseEntity;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.util.List;
/**
* 描述：
* 用户聊天管理Service接口
*
* @author https:www.unfbx.com
* @date 2023-04-08
* @author ageerle
* @date 2025-04-08
*/
public interface ISseService {
/**
* 客户端发送消息到服务端
* @param chatRequest
* @param chatRequest 请求对象
*/
SseEmitter sseChat(ChatRequest chatRequest,HttpServletRequest request);
SseEmitter sseChat(ChatRequest chatRequest, HttpServletRequest request);
/**
* 语音转文字
* @param file
* @param file 语音文件
*/
WhisperResponse speechToTextTranscriptionsV2(MultipartFile file);
/**
* 文字转语音
*
* @param textToSpeech 文本信息
* @return 流式语音
*/
ResponseEntity<Resource> textToSpeed(TextToSpeech textToSpeech);
/**
* 客户端发送消息到服务端
* @param chatRequest
*/
String chat(ChatRequest chatRequest,String userId);
/**
* 客户端发送消息到服务端
* 上传文件到api服务器
*
* @param file 文件信息
* @return 返回文件信息
*/
List<Item> wxDall(String prompt,String userId);
/**
* 绘画接口
* @param request
*/
List<Item> dall3(Dall3Request request);
UploadFileResponse upload(MultipartFile file);
/**
* 文本翻译
* @param
*/
String translation(TranslationRequest translationRequest);
/**
* 调用本地模型
* @param chatRequest
* 使用ollama调用本地模型
* @param chatRequest 对话信息
* @return 流式输出返回内容
*/
SseEmitter ollamaChat(ChatRequest chatRequest);

292
ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java Normal file
View File

@@ -0,0 +1,292 @@
package org.ruoyi.chat.service.chat.impl;
import cn.dev33.satoken.stp.StpUtil;
import cn.hutool.extra.servlet.ServletUtil;
import com.google.protobuf.ServiceException;
import io.github.ollama4j.OllamaAPI;
import io.github.ollama4j.models.chat.OllamaChatMessage;
import io.github.ollama4j.models.chat.OllamaChatMessageRole;
import io.github.ollama4j.models.chat.OllamaChatRequestBuilder;
import io.github.ollama4j.models.chat.OllamaChatRequestModel;
import io.github.ollama4j.models.generate.OllamaStreamHandler;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import org.ruoyi.chat.listener.SSEEventSourceListener;
import org.ruoyi.chat.service.chat.ISseService;
import org.ruoyi.common.chat.config.ChatConfig;
import org.ruoyi.common.chat.domain.request.ChatRequest;
import org.ruoyi.common.chat.entity.Tts.TextToSpeech;
import org.ruoyi.common.chat.entity.chat.ChatCompletion;
import org.ruoyi.common.chat.entity.chat.ChatCompletionResponse;
import org.ruoyi.common.chat.entity.chat.Message;
import org.ruoyi.common.chat.entity.files.UploadFileResponse;
import org.ruoyi.common.chat.entity.whisper.WhisperResponse;
import org.ruoyi.common.chat.openai.OpenAiStreamClient;
import org.ruoyi.common.core.utils.file.FileUtils;
import org.ruoyi.common.core.utils.file.MimeTypeUtils;
import org.ruoyi.common.redis.utils.RedisUtils;
import org.ruoyi.domain.vo.ChatModelVo;
import org.ruoyi.service.IChatModelService;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@Service
@Slf4j
@RequiredArgsConstructor
public class SseServiceImpl implements ISseService {
private OpenAiStreamClient openAiStreamClient;
private final ChatConfig chatConfig;
private final IChatModelService chatModelService;
@Override
public SseEmitter sseChat(ChatRequest chatRequest, HttpServletRequest request) {
SseEmitter sseEmitter = new SseEmitter(0L);
SSEEventSourceListener openAIEventSourceListener = new SSEEventSourceListener(sseEmitter);
// 获取对话消息列表
List<Message> messages = chatRequest.getMessages();
try {
if (StpUtil.isLogin()) {
// 通过模型名称查询模型信息
ChatModelVo chatModelVo = chatModelService.selectModelByName(chatRequest.getModel());
// 构建api请求客户端
openAiStreamClient = chatConfig.createOpenAiStreamClient(chatModelVo.getApiHost(), chatModelVo.getApiKey());
// 模型设置默认提示词
// 是否开启联网查询
}else {
// 未登录用户限制对话次数,默认5次
String clientIp = ServletUtil.getClientIP((javax.servlet.http.HttpServletRequest) request,"X-Forwarded-For");
int timeWindowInSeconds = 5;
String redisKey = "visitor:" + clientIp;
int count = 0;
if (RedisUtils.getCacheObject(redisKey) == null) {
// 当前访问次数
RedisUtils.setCacheObject(redisKey, count, Duration.ofSeconds(86400));
}else {
count = RedisUtils.getCacheObject(redisKey);
if (count >= timeWindowInSeconds) {
throw new ServiceException("当日免费次数已用完");
}
count++;
RedisUtils.setCacheObject(redisKey, count);
}
}
ChatCompletion completion = ChatCompletion
.builder()
.messages(messages)
.model(chatRequest.getModel())
.temperature(chatRequest.getTemperature())
.topP(chatRequest.getTop_p())
.stream(true)
.build();
openAiStreamClient.streamChatCompletion(completion, openAIEventSourceListener);
// 保存消息记录 并扣除费用
} catch (Exception e) {
String message = e.getMessage();
sendErrorEvent(sseEmitter, message);
return sseEmitter;
}
return sseEmitter;
}
/**
* 发送SSE错误事件的封装方法
*
* @param sseEmitter
* @param errorMessage
*/
private void sendErrorEvent(SseEmitter sseEmitter, String errorMessage) {
SseEmitter.SseEventBuilder event = SseEmitter.event()
.name("error")
.data(errorMessage);
try {
sseEmitter.send(event);
} catch (IOException e) {
log.error("发送事件失败: {}", e.getMessage());
}
sseEmitter.complete();
}
/**
* 文字转语音
*/
@Override
public ResponseEntity<Resource> textToSpeed(TextToSpeech textToSpeech) {
ResponseBody body = openAiStreamClient.textToSpeech(textToSpeech);
if (body != null) {
// 将ResponseBody转换为InputStreamResource
InputStreamResource resource = new InputStreamResource(body.byteStream());
// 创建并返回ResponseEntity
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType("audio/mpeg"))
.body(resource);
} else {
// 如果ResponseBody为空，返回404状态码
return ResponseEntity.notFound().build();
}
}
/**
* 语音转文字
*/
@Override
public WhisperResponse speechToTextTranscriptionsV2(MultipartFile file) {
// 确保文件不为空
if (file.isEmpty()) {
throw new IllegalStateException("Cannot convert an empty MultipartFile");
}
if (!FileUtils.isValidFileExtention(file, MimeTypeUtils.AUDIO__EXTENSION)) {
throw new IllegalStateException("File Extention not supported");
}
// 创建一个文件对象
File fileA = new File(System.getProperty("java.io.tmpdir") + File.separator + file.getOriginalFilename());
try {
// 将 MultipartFile 的内容写入文件
file.transferTo(fileA);
} catch (IOException e) {
throw new RuntimeException("Failed to convert MultipartFile to File", e);
}
return openAiStreamClient.speechToTextTranscriptions(fileA);
}
@Override
public UploadFileResponse upload(MultipartFile file) {
if (file.isEmpty()) {
throw new IllegalStateException("Cannot upload an empty MultipartFile");
}
if (!FileUtils.isValidFileExtention(file, MimeTypeUtils.DEFAULT_ALLOWED_EXTENSION)) {
throw new IllegalStateException("File Extention not supported");
}
openAiStreamClient = chatConfig.getOpenAiStreamClient();
return openAiStreamClient.uploadFile("fine-tune", convertMultiPartToFile(file));
}
private File convertMultiPartToFile(MultipartFile multipartFile) {
File file = null;
try {
// 获取原始文件名
String originalFileName = multipartFile.getOriginalFilename();
// 默认扩展名
String extension = ".tmp";
// 尝试从原始文件名中获取扩展名
if (originalFileName != null && originalFileName.contains(".")) {
extension = originalFileName.substring(originalFileName.lastIndexOf("."));
}
// 使用原始文件的扩展名创建临时文件
Path tempFile = Files.createTempFile(null, extension);
file = tempFile.toFile();
// 将MultipartFile的内容写入文件
try (InputStream inputStream = multipartFile.getInputStream();
FileOutputStream outputStream = new FileOutputStream(file)) {
int read;
byte[] bytes = new byte[1024];
while ((read = inputStream.read(bytes)) != -1) {
outputStream.write(bytes, 0, read);
}
} catch (IOException e) {
// 处理文件写入异常
e.printStackTrace();
}
} catch (IOException e) {
// 处理临时文件创建异常
e.printStackTrace();
}
return file;
}
@Override
public SseEmitter ollamaChat(ChatRequest chatRequest) {
String[] parts = chatRequest.getModel().split("ollama-");
ChatModelVo chatModelVo = chatModelService.selectModelByName(chatRequest.getModel());
final SseEmitter emitter = new SseEmitter();
String host = chatModelVo.getApiHost();
List<Message> msgList = chatRequest.getMessages();
List<OllamaChatMessage> messages = new ArrayList<>();
for (Message message : msgList) {
OllamaChatMessage ollamaChatMessage = new OllamaChatMessage();
ollamaChatMessage.setRole(OllamaChatMessageRole.USER);
ollamaChatMessage.setContent(message.getContent().toString());
messages.add(ollamaChatMessage);
}
OllamaAPI api = new OllamaAPI(host);
api.setRequestTimeoutSeconds(100);
OllamaChatRequestBuilder builder = OllamaChatRequestBuilder.getInstance(parts[1]);
OllamaChatRequestModel requestModel = builder
.withMessages(messages)
.build();
// 异步执行 OllAma API 调用
CompletableFuture.runAsync(() -> {
try {
StringBuilder response = new StringBuilder();
OllamaStreamHandler streamHandler = (s) -> {
String substr = s.substring(response.length());
response.append(substr);
System.out.println(substr);
try {
emitter.send(substr);
} catch (IOException e) {
sendErrorEvent(emitter, e.getMessage());
}
};
api.chat(requestModel, streamHandler);
emitter.complete();
} catch (Exception e) {
sendErrorEvent(emitter, e.getMessage());
}
});
return emitter;
}
@Override
public String wxCpChat(String prompt) {
List<Message> messageList = new ArrayList<>();
Message message = Message.builder().role(Message.Role.USER).content(prompt).build();
messageList.add(message);
ChatCompletion chatCompletion = ChatCompletion
.builder()
.messages(messageList)
.model("gpt-4o-mini")
.stream(false)
.build();
ChatCompletionResponse chatCompletionResponse = openAiStreamClient.chatCompletion(chatCompletion);
return chatCompletionResponse.getChoices().get(0).getMessage().getContent().toString();
}
}

17
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/BgeLargeVectorization.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/vectorizer/BgeLargeVectorization.java
View File

@@ -1,8 +1,4 @@
package org.ruoyi.knowledge.chain.vectorizer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
package org.ruoyi.chat.service.knowledge.vectorizer;
import com.google.gson.Gson;
import io.github.ollama4j.OllamaAPI;
@@ -10,15 +6,20 @@ import io.github.ollama4j.models.embeddings.OllamaEmbeddingsRequestModel;
import jakarta.annotation.Resource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.ruoyi.domain.vo.KnowledgeInfoVo;
import org.ruoyi.service.IKnowledgeInfoService;
import org.ruoyi.service.VectorizationService;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Component
@Slf4j
@RequiredArgsConstructor
public class BgeLargeVectorization implements Vectorization {
public class BgeLargeVectorization implements VectorizationService {
String host = "http://localhost:11434/";

15
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/vectorizer/OpenAiVectorization.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.knowledge.chain.vectorizer;
package org.ruoyi.chat.service.knowledge.vectorizer;
import jakarta.annotation.Resource;
import lombok.Getter;
@@ -6,14 +6,13 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.chat.config.ChatConfig;
import org.ruoyi.common.chat.entity.embeddings.Embedding;
import org.ruoyi.common.chat.entity.embeddings.EmbeddingResponse;
import org.ruoyi.common.chat.openai.OpenAiStreamClient;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.ruoyi.domain.vo.KnowledgeInfoVo;
import org.ruoyi.service.IKnowledgeInfoService;
import org.ruoyi.service.VectorizationService;
import org.ruoyi.system.domain.SysModel;
import org.ruoyi.system.service.ISysModelService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@@ -25,14 +24,12 @@ import java.util.stream.Collectors;
@Component
@Slf4j
@RequiredArgsConstructor
public class OpenAiVectorization implements Vectorization {
public class OpenAiVectorization implements VectorizationService {
@Lazy
@Resource
private IKnowledgeInfoService knowledgeInfoService;
@Lazy
@Resource
private LocalModelsVectorization localModelsVectorization;
@Lazy
@Resource
private ISysModelService sysModelService;

12
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationFactory.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/vectorizer/VectorizationFactory.java
View File

@@ -1,10 +1,12 @@
package org.ruoyi.knowledge.chain.vectorizer;
package org.ruoyi.chat.service.knowledge.vectorizer;
import cn.hutool.core.util.StrUtil;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.ruoyi.domain.vo.KnowledgeInfoVo;
import org.ruoyi.service.IKnowledgeInfoService;
import org.ruoyi.service.VectorizationService;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@@ -24,12 +26,12 @@ public class VectorizationFactory {
@Resource
private IKnowledgeInfoService knowledgeInfoService;
public VectorizationFactory(OpenAiVectorization openAiVectorization,BgeLargeVectorization bgeLargeVectorization) {
public VectorizationFactory(OpenAiVectorization openAiVectorization, BgeLargeVectorization bgeLargeVectorization) {
this.openAiVectorization = openAiVectorization;
this.bgeLargeVectorization = bgeLargeVectorization;
}
public Vectorization getEmbedding(String kid){
public VectorizationService getEmbedding(String kid){
String vectorModel = "text-embedding-3-small";
if (StrUtil.isNotEmpty(kid)) {
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));

7
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/vectorstore/MilvusVectorStore.java
View File

@@ -1,4 +1,4 @@
package org.ruoyi.knowledge.chain.vectorstore;
package org.ruoyi.chat.service.knowledge.vectorstore;
import io.milvus.client.MilvusServiceClient;
import io.milvus.grpc.DataType;
@@ -20,7 +20,8 @@ import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.ruoyi.common.core.service.ConfigService;
import org.springframework.beans.factory.annotation.Value;
import org.ruoyi.service.VectorStoreService;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@@ -29,7 +30,7 @@ import java.util.List;
@Service
@Slf4j
public class MilvusVectorStore implements VectorStore {
public class MilvusVectorStore implements VectorStoreService {
private volatile Integer dimension;
private volatile String collectionName;

9
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/vectorstore/VectorStoreFactory.java
View File

@@ -1,10 +1,11 @@
package org.ruoyi.knowledge.chain.vectorstore;
package org.ruoyi.chat.service.knowledge.vectorstore;
import cn.hutool.core.util.StrUtil;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper;
import org.ruoyi.domain.vo.KnowledgeInfoVo;
import org.ruoyi.mapper.KnowledgeInfoMapper;
import org.ruoyi.service.VectorStoreService;
import org.springframework.stereotype.Component;
@Component
@@ -23,7 +24,7 @@ public class VectorStoreFactory {
this.milvusVectorStore = milvusVectorStore;
}
public VectorStore getVectorStore(String kid){
public VectorStoreService getVectorStore(String kid){
String vectorModel = "weaviate";
if (StrUtil.isNotEmpty(kid)) {
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoMapper.selectVoById(Long.valueOf(kid));

12
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java → ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/knowledge/vectorstore/WeaviateVectorStore.java
View File

@@ -1,7 +1,7 @@
package org.ruoyi.knowledge.chain.vectorstore;
package org.ruoyi.chat.service.knowledge.vectorstore;
import cn.hutool.core.lang.UUID;
import com.alibaba.fastjson2.JSONObject;
import cn.hutool.json.JSONObject;
import com.google.gson.internal.LinkedTreeMap;
import io.weaviate.client.Config;
import io.weaviate.client.WeaviateClient;
@@ -27,9 +27,9 @@ import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.ruoyi.common.core.service.ConfigService;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.beans.factory.annotation.Value;
import org.ruoyi.domain.vo.KnowledgeInfoVo;
import org.ruoyi.service.IKnowledgeInfoService;
import org.ruoyi.service.VectorStoreService;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
@@ -40,7 +40,7 @@ import java.util.Map;
@Service
@Slf4j
public class WeaviateVectorStore implements VectorStore {
public class WeaviateVectorStore implements VectorStoreService {
private volatile String protocol;
private volatile String host;

21
ruoyi-modules/ruoyi-device/pom.xml
View File

@@ -1,21 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.ruoyi</groupId>
<artifactId>ruoyi-ai</artifactId>
<version>1.0.0</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>ruoyi-device</artifactId>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
</project>

203
ruoyi-modules/ruoyi-knowledge/pom.xml
View File

@@ -1,203 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.ruoyi</groupId>
<artifactId>ruoyi-modules</artifactId>
<version>${revision}</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>ruoyi-knowledge</artifactId>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-freemarker</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>org.webjars</groupId>
<artifactId>webjars-locator-core</artifactId>
</dependency>
<dependency>
<groupId>org.webjars</groupId>
<artifactId>sockjs-client</artifactId>
<version>1.0.2</version>
</dependency>
<dependency>
<groupId>org.webjars</groupId>
<artifactId>bootstrap</artifactId>
<version>3.3.7</version>
</dependency>
<dependency>
<groupId>org.webjars</groupId>
<artifactId>jquery</artifactId>
<version>3.1.1-1</version>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>3.0.2</version>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>3.5.3</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.10</version>
</dependency>
<dependency>
<groupId>com.github.xiaoymin</groupId>
<artifactId>knife4j-openapi3-jakarta-spring-boot-starter</artifactId>
<version>4.0.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty-core</artifactId>
<version>1.1.16</version>
</dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty-http</artifactId>
<version>1.1.16</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.theokanning.openai-gpt3-java</groupId>
<artifactId>api</artifactId>
<version>0.18.0</version>
</dependency>
<dependency>
<groupId>com.theokanning.openai-gpt3-java</groupId>
<artifactId>client</artifactId>
<version>0.18.0</version>
</dependency>
<dependency>
<groupId>com.theokanning.openai-gpt3-java</groupId>
<artifactId>service</artifactId>
<version>0.18.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<!-- 阿里JSON解析器 -->
<dependency>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2</artifactId>
<version>2.0.24</version>
</dependency>
<dependency>
<groupId>com.knuddels</groupId>
<artifactId>jtokkit</artifactId>
<version>0.5.0</version>
</dependency>
<dependency>
<groupId>io.weaviate</groupId>
<artifactId>client</artifactId>
<version>4.0.0</version> <!-- Check latest version -->
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>3.8</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.8</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml-schemas</artifactId>
<version>3.8</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>2.0.27</version>
</dependency>
<dependency>
<groupId>com.squareup.retrofit2</groupId>
<artifactId>converter-jackson</artifactId>
<version>2.9.0</version>
</dependency>
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
<version>2.3.2</version>
</dependency>
<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper-spring-boot-starter</artifactId>
<version>1.4.6</version>
</dependency>
<dependency>
<groupId>com.belerweb</groupId>
<artifactId>pinyin4j</artifactId>
<version>2.5.1</version>
</dependency>
<dependency>
<groupId>org.ruoyi</groupId>
<artifactId>ruoyi-system</artifactId>
</dependency>
</dependencies>
</project>

37
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java
View File

@@ -1,37 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.knowledge.chain.split.TextSplitter;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class CodeFileLoader implements ResourceLoader{
private final TextSplitter textSplitter;
@Override
public String getContent(InputStream inputStream) {
StringBuffer stringBuffer = new StringBuffer();
try (InputStreamReader reader = new InputStreamReader(inputStream);
BufferedReader bufferedReader = new BufferedReader(reader)){
String line;
while ((line = bufferedReader.readLine()) != null) {
stringBuffer.append(line).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
}
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content, String kid){
return textSplitter.split(content, kid);
}
}

16
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java
View File

@@ -1,16 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import java.io.InputStream;
import java.util.List;
public class CsvFileLoader implements ResourceLoader{
@Override
public String getContent(InputStream inputStream) {
return null;
}
@Override
public List<String> getChunkList(String content, String kid) {
return null;
}
}

16
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java
View File

@@ -1,16 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import java.io.InputStream;
import java.util.List;
public class FolderLoader implements ResourceLoader{
@Override
public String getContent(InputStream inputStream) {
return null;
}
@Override
public List<String> getChunkList(String content, String kid) {
return null;
}
}

16
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java
View File

@@ -1,16 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import java.io.InputStream;
import java.util.List;
public class GithubLoader implements ResourceLoader{
@Override
public String getContent(InputStream inputStream) {
return null;
}
@Override
public List<String> getChunkList(String content, String kid) {
return null;
}
}

16
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java
View File

@@ -1,16 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import java.io.InputStream;
import java.util.List;
public class JsonFileLoader implements ResourceLoader{
@Override
public String getContent(InputStream inputStream) {
return null;
}
@Override
public List<String> getChunkList(String content, String kid) {
return null;
}
}

37
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java
View File

@@ -1,37 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.knowledge.chain.split.TextSplitter;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class MarkDownFileLoader implements ResourceLoader{
private final TextSplitter textSplitter;
@Override
public String getContent(InputStream inputStream) {
StringBuffer stringBuffer = new StringBuffer();
try (InputStreamReader reader = new InputStreamReader(inputStream);
BufferedReader bufferedReader = new BufferedReader(reader)){
String line;
while ((line = bufferedReader.readLine()) != null) {
stringBuffer.append(line).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
}
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content, String kid){
return textSplitter.split(content, kid);
}
}

34
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java
View File

@@ -1,34 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import lombok.AllArgsConstructor;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import org.ruoyi.knowledge.chain.split.TextSplitter;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
@Component
@AllArgsConstructor
public class PdfFileLoader implements ResourceLoader{
private final TextSplitter characterTextSplitter;
@Override
public String getContent(InputStream inputStream) {
PDDocument document = null;
try {
document = PDDocument.load(inputStream);
PDFTextStripper textStripper = new PDFTextStripper();
String content = textStripper.getText(document);
return content;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public List<String> getChunkList(String content, String kid) {
return characterTextSplitter.split(content, kid);
}
}

14
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java
View File

@@ -1,14 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import java.io.InputStream;
import java.util.List;
/**
* 资源载入
*/
public interface ResourceLoader {
String getContent(InputStream inputStream);
List<String> getChunkList(String content, String kid);
}

33
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoaderFactory.java
View File

@@ -1,33 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import lombok.AllArgsConstructor;
import org.ruoyi.knowledge.chain.split.CharacterTextSplitter;
import org.ruoyi.knowledge.chain.split.CodeTextSplitter;
import org.ruoyi.knowledge.chain.split.MarkdownTextSplitter;
import org.ruoyi.knowledge.chain.split.TokenTextSplitter;
import org.ruoyi.knowledge.constant.FileType;
import org.springframework.stereotype.Component;
@AllArgsConstructor
@Component
public class ResourceLoaderFactory {
private final CharacterTextSplitter characterTextSplitter;
private final CodeTextSplitter codeTextSplitter;
private final MarkdownTextSplitter markdownTextSplitter;
private final TokenTextSplitter tokenTextSplitter;
public ResourceLoader getLoaderByFileType(String fileType){
if (FileType.isTextFile(fileType)){
return new TextFileLoader(characterTextSplitter);
} else if (FileType.isWord(fileType)) {
return new WordLoader(characterTextSplitter);
} else if (FileType.isPdf(fileType)) {
return new PdfFileLoader(characterTextSplitter);
} else if (FileType.isMdFile(fileType)) {
return new MarkDownFileLoader(markdownTextSplitter);
}else if (FileType.isCodeFile(fileType)) {
return new CodeFileLoader(codeTextSplitter);
}else {
return new TextFileLoader(characterTextSplitter);
}
}
}

37
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java
View File

@@ -1,37 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.knowledge.chain.split.TextSplitter;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class TextFileLoader implements ResourceLoader{
private final TextSplitter textSplitter;
@Override
public String getContent(InputStream inputStream) {
StringBuffer stringBuffer = new StringBuffer();
try (InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
BufferedReader bufferedReader = new BufferedReader(reader)){
String line;
while ((line = bufferedReader.readLine()) != null) {
stringBuffer.append(line).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
}
return stringBuffer.toString();
}
@Override
public List<String> getChunkList(String content, String kid){
return textSplitter.split(content, kid);
}
}

37
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java
View File

@@ -1,37 +0,0 @@
package org.ruoyi.knowledge.chain.loader;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.ruoyi.knowledge.chain.split.TextSplitter;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class WordLoader implements ResourceLoader{
private final TextSplitter textSplitter;
@Override
public String getContent(InputStream inputStream) {
XWPFDocument document = null;
try {
document = new XWPFDocument(inputStream);
XWPFWordExtractor extractor = new XWPFWordExtractor(document);
String content = extractor.getText();
return content;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public List<String> getChunkList(String content, String kid) {
return textSplitter.split(content, kid);
}
}

63
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java
View File

@@ -1,63 +0,0 @@
package org.ruoyi.knowledge.chain.split;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Component
@Slf4j
@Primary
public class CharacterTextSplitter implements TextSplitter {
@Lazy
@Resource
private IKnowledgeInfoService knowledgeInfoService;
@Override
public List<String> split(String content, String kid) {
// 从知识库表中获取配置
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));
String knowledgeSeparator = knowledgeInfoVo.getKnowledgeSeparator();
int textBlockSize = knowledgeInfoVo.getTextBlockSize();
int overlapChar = knowledgeInfoVo.getOverlapChar();
List<String> chunkList = new ArrayList<>();
if (content.contains(knowledgeSeparator) && StringUtils.isNotBlank(knowledgeSeparator)) {
// 按自定义分隔符切分
String[] chunks = content.split(knowledgeSeparator);
chunkList.addAll(Arrays.asList(chunks));
} else {
int indexMin = 0;
int len = content.length();
int i = 0;
int right = 0;
while (true) {
if (len > right) {
int begin = i * textBlockSize - overlapChar;
if (begin < indexMin) {
begin = indexMin;
}
int end = textBlockSize * (i + 1) + overlapChar;
if (end > len) {
end = len;
}
String chunk = content.substring(begin, end);
chunkList.add(chunk);
i++;
right = right + textBlockSize;
} else {
break;
}
}
}
return chunkList;
}
}

17
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java
View File

@@ -1,17 +0,0 @@
package org.ruoyi.knowledge.chain.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class CodeTextSplitter implements TextSplitter{
@Override
public List<String> split(String content, String kid) {
return null;
}
}

17
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java
View File

@@ -1,17 +0,0 @@
package org.ruoyi.knowledge.chain.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class MarkdownTextSplitter implements TextSplitter{
@Override
public List<String> split(String content, String kid) {
return null;
}
}

18
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java
View File

@@ -1,18 +0,0 @@
package org.ruoyi.knowledge.chain.split;
import java.util.List;
/**
* 文本切分
*/
public interface TextSplitter {
/**
* 文本切分
*
* @param content 文本内容
* @param kid 知识库id
* @return 切分后的文本列表
*/
List<String> split(String content, String kid);
}

17
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java
View File

@@ -1,17 +0,0 @@
package org.ruoyi.knowledge.chain.split;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@AllArgsConstructor
@Slf4j
public class TokenTextSplitter implements TextSplitter{
@Override
public List<String> split(String content, String kid) {
return null;
}
}

92
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/LocalModelsVectorization.java
View File

@@ -1,92 +0,0 @@
package org.ruoyi.knowledge.chain.vectorizer;
import jakarta.annotation.Resource;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.common.chat.config.ChatConfig;
import org.ruoyi.common.chat.localModels.LocalModelsofitClient;
import org.ruoyi.common.chat.openai.OpenAiStreamClient;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
@Component
@Slf4j
@RequiredArgsConstructor
public class LocalModelsVectorization {
@Resource
private IKnowledgeInfoService knowledgeInfoService;
@Resource
private LocalModelsofitClient localModelsofitClient;
@Getter
private OpenAiStreamClient openAiStreamClient;
private final ChatConfig chatConfig;
/**
* 批量向量化
*
* @param chunkList 文本块列表
* @param kid 知识 ID
* @return 向量化结果
*/
public List<List<Double>> batchVectorization(List<String> chunkList, String kid) {
logVectorizationRequest(kid, chunkList); // 在向量化开始前记录日志
openAiStreamClient = chatConfig.getOpenAiStreamClient(); // 获取 OpenAi 客户端
KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid)); // 查询知识信息
// 调用 localModelsofitClient 获取 Top K 嵌入向量
try {
return localModelsofitClient.getTopKEmbeddings(
chunkList,
knowledgeInfoVo.getVectorModel(),
knowledgeInfoVo.getKnowledgeSeparator(),
knowledgeInfoVo.getRetrieveLimit(),
knowledgeInfoVo.getTextBlockSize(),
knowledgeInfoVo.getOverlapChar()
);
} catch (Exception e) {
log.error("Failed to perform batch vectorization for knowledgeId: {}", kid, e);
throw new RuntimeException("Batch vectorization failed", e);
}
}
/**
* 单一文本块向量化
*
* @param chunk 单一文本块
* @param kid 知识 ID
* @return 向量化结果
*/
public List<Double> singleVectorization(String chunk, String kid) {
List<String> chunkList = new ArrayList<>();
chunkList.add(chunk);
// 调用批量向量化方法
List<List<Double>> vectorList = batchVectorization(chunkList, kid);
if (vectorList.isEmpty()) {
log.warn("Vectorization returned empty list for chunk: {}", chunk);
return new ArrayList<>();
}
return vectorList.get(0); // 返回第一个向量
}
/**
* 提供更简洁的日志记录方法
*
* @param kid 知识 ID
* @param chunkList 文本块列表
*/
private void logVectorizationRequest(String kid, List<String> chunkList) {
log.info("Starting vectorization for Knowledge ID: {} with {} chunks.", kid, chunkList.size());
}
}

12
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java
View File

@@ -1,12 +0,0 @@
package org.ruoyi.knowledge.chain.vectorizer;
import java.util.List;
/**
* 向量化
*/
public interface Vectorization {
List<List<Double>> batchVectorization(List<String> chunkList, String kid);
List<Double> singleVectorization(String chunk, String kid);
}

15
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationType.java
View File

@@ -1,15 +0,0 @@
package org.ruoyi.knowledge.chain.vectorizer;
public enum VectorizationType {
OPENAI, // OpenAI 向量化
LOCAL; // 本地模型向量化
public static VectorizationType fromString(String type) {
for (VectorizationType v : values()) {
if (v.name().equalsIgnoreCase(type)) {
return v;
}
}
throw new IllegalArgumentException("Unknown VectorizationType: " + type);
}
}

32
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java
View File

@@ -1,32 +0,0 @@
package org.ruoyi.knowledge.chain.vectorizer;
import jakarta.annotation.Resource;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@Slf4j
@Primary
@AllArgsConstructor
public class VectorizationWrapper implements Vectorization{
private final VectorizationFactory vectorizationFactory;
@Override
public List<List<Double>> batchVectorization(List<String> chunkList, String kid) {
Vectorization embedding = vectorizationFactory.getEmbedding(kid);
return embedding.batchVectorization(chunkList, kid);
}
@Override
public List<Double> singleVectorization(String chunk, String kid) {
Vectorization embedding = vectorizationFactory.getEmbedding(kid);
return embedding.singleVectorization(chunk, kid);
}
}

23
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStore.java
View File

@@ -1,23 +0,0 @@
package org.ruoyi.knowledge.chain.vectorstore;
import java.util.List;
/**
* 向量存储
*/
public interface VectorStore {
void storeEmbeddings(List<String> chunkList, List<List<Double>> vectorList, String kid, String docId, List<String> fidList);
void removeByDocId(String kid, String docId);
void removeByKid(String kid);
List<String> nearest(List<Double> queryVector, String kid);
List<String> nearest(String query, String kid);
void newSchema(String kid);
void removeByKidAndFid(String kid, String fid);
}

59
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java
View File

@@ -1,59 +0,0 @@
package org.ruoyi.knowledge.chain.vectorstore;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
@Slf4j
@Primary
@AllArgsConstructor
public class VectorStoreWrapper implements VectorStore {
private final VectorStoreFactory vectorStoreFactory;
@Override
public void storeEmbeddings(List<String> chunkList, List<List<Double>> vectorList, String kid, String docId, List<String> fidList) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.storeEmbeddings(chunkList, vectorList, kid, docId, fidList);
}
@Override
public void removeByDocId(String kid, String docId) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.removeByDocId(kid, docId);
}
@Override
public void removeByKid(String kid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.removeByKid(kid);
}
@Override
public List<String> nearest(List<Double> queryVector, String kid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
return vectorStore.nearest(queryVector, kid);
}
@Override
public List<String> nearest(String query, String kid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
return vectorStore.nearest(query, kid);
}
@Override
public void newSchema(String kid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.newSchema(kid);
}
@Override
public void removeByKidAndFid(String kid, String fid) {
VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.removeByKidAndFid(kid, fid);
}
}

91
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/constant/FileType.java
View File

@@ -1,91 +0,0 @@
package org.ruoyi.knowledge.constant;
public class FileType {
public static final String TXT = "txt";
public static final String CSV = "csv";
public static final String MD = "md";
public static final String DOC = "doc";
public static final String DOCX = "docx";
public static final String PDF = "pdf";
public static final String LOG = "log";
public static final String XML = "xml";
public static final String JAVA = "java";
public static final String HTML = "html";
public static final String HTM = "htm";
public static final String CSS = "css";
public static final String JS = "js";
public static final String PY = "py";
public static final String CPP = "cpp";
public static final String SQL = "sql";
public static final String PHP = "php";
public static final String RUBY = "ruby";
public static final String C = "c";
public static final String H = "h";
public static final String HPP = "hpp";
public static final String SWIFT = "swift";
public static final String TS = "ts";
public static final String RUST = "rs";
public static final String PERL = "perl";
public static final String SHELL = "shell";
public static final String BAT = "bat";
public static final String CMD = "cmd";
public static final String PROPERTIES = "properties";
public static final String INI = "ini";
public static final String YAML = "yaml";
public static final String YML = "yml";
public static boolean isTextFile(String type){
if (type.equalsIgnoreCase(TXT) || type.equalsIgnoreCase(CSV) || type.equalsIgnoreCase(PROPERTIES)
|| type.equalsIgnoreCase(INI) || type.equalsIgnoreCase(YAML) || type.equalsIgnoreCase(YML)
|| type.equalsIgnoreCase(LOG) || type.equalsIgnoreCase(XML)){
return true;
}
else {
return false;
}
}
public static boolean isCodeFile(String type){
if (type.equalsIgnoreCase(JAVA) || type.equalsIgnoreCase(HTML) || type.equalsIgnoreCase(HTM) || type.equalsIgnoreCase(JS) || type.equalsIgnoreCase(PY)
|| type.equalsIgnoreCase(CPP) || type.equalsIgnoreCase(SQL) || type.equalsIgnoreCase(PHP) || type.equalsIgnoreCase(RUBY)
|| type.equalsIgnoreCase(C) || type.equalsIgnoreCase(H) || type.equalsIgnoreCase(HPP) || type.equalsIgnoreCase(SWIFT)
|| type.equalsIgnoreCase(TS) || type.equalsIgnoreCase(RUST) || type.equalsIgnoreCase(PERL) || type.equalsIgnoreCase(SHELL)
|| type.equalsIgnoreCase(BAT) || type.equalsIgnoreCase(CMD) || type.equalsIgnoreCase(CSS)){
return true;
}
else {
return false;
}
}
public static boolean isMdFile(String type){
if (type.equalsIgnoreCase(MD)){
return true;
}
else {
return false;
}
}
public static boolean isWord(String type){
if (type.equalsIgnoreCase(DOC) || type.equalsIgnoreCase(DOCX)){
return true;
}
else {
return false;
}
}
public static boolean isPdf(String type){
if (type.equalsIgnoreCase(PDF)){
return true;
}
else {
return false;
}
}
}

Some files were not shown because too many files have changed in this diff Show More